QueryServer: Use non-destructive upgrades where possible.

This commit is contained in:
alexet
2020-11-13 16:21:02 +00:00
committed by Andrew Eisenberg
parent cb4d6f228b
commit a25db9616f
4 changed files with 251 additions and 216 deletions

View File

@@ -12,12 +12,10 @@ import {
} from 'vscode';
import * as fs from 'fs-extra';
import * as cli from './cli';
import {
DatabaseChangedEvent,
DatabaseItem,
DatabaseManager,
getUpgradesDirectories,
} from './databases';
import {
commandRunner,
@@ -33,7 +31,7 @@ import {
import { logger } from './logging';
import { clearCacheInDatabase } from './run-queries';
import * as qsClient from './queryserver-client';
import { upgradeDatabase } from './upgrades';
import { upgradeDatabaseExplicit } from './upgrades';
import {
importArchiveDatabase,
promptImportInternetDatabase,
@@ -218,7 +216,6 @@ export class DatabaseUI extends DisposableObject {
private treeDataProvider: DatabaseTreeDataProvider;
public constructor(
private cliserver: cli.CodeQLCliServer,
private databaseManager: DatabaseManager,
private readonly queryServer: qsClient.QueryServerClient | undefined,
private readonly storagePath: string,
@@ -540,25 +537,10 @@ export class DatabaseUI extends DisposableObject {
}
// Search for upgrade scripts in any workspace folders available
const searchPath: string[] = getOnDiskWorkspaceFolders();
const upgradeInfo = await this.cliserver.resolveUpgrades(
databaseItem.contents.dbSchemeUri.fsPath,
searchPath
);
const { scripts, finalDbscheme } = upgradeInfo;
if (finalDbscheme === undefined) {
throw new Error('Could not determine target dbscheme to upgrade to.');
}
const targetDbSchemeUri = Uri.file(finalDbscheme);
await upgradeDatabase(
await upgradeDatabaseExplicit(
this.queryServer,
databaseItem,
targetDbSchemeUri,
getUpgradesDirectories(scripts),
progress,
token
);

View File

@@ -369,7 +369,6 @@ async function activateWithInstalledDistribution(
ctx.subscriptions.push(dbm);
logger.log('Initializing database panel.');
const databaseUI = new DatabaseUI(
cliServer,
dbm,
qs,
getContextStoragePath(ctx),

View File

@@ -14,16 +14,17 @@ import { ErrorCodes, ResponseError } from 'vscode-languageclient';
import * as cli from './cli';
import * as config from './config';
import { DatabaseItem, getUpgradesDirectories } from './databases';
import { DatabaseItem } from './databases';
import { getOnDiskWorkspaceFolders, showAndLogErrorMessage } from './helpers';
import { ProgressCallback, UserCancellationException } from './commandRunner';
import * as helpers from './helpers';
import { DatabaseInfo, QueryMetadata, ResultsPaths } from './pure/interface-types';
import { logger } from './logging';
import * as messages from './pure/messages';
import { QueryHistoryItemOptions } from './query-history';
import * as qsClient from './queryserver-client';
import { isQuickQueryPath } from './quick-query';
import { upgradeDatabase } from './upgrades';
import { compileDatabaseUpgradeSequence, hasNondestructiveUpgradeCapabilities, upgradeDatabaseExplicit } from './upgrades';
/**
* run-queries.ts
@@ -80,6 +81,7 @@ export class QueryInfo {
async run(
qs: qsClient.QueryServerClient,
upgradeQlo: string | undefined,
progress: ProgressCallback,
token: CancellationToken,
): Promise<messages.EvaluationResult> {
@@ -90,6 +92,7 @@ export class QueryInfo {
const queryToRun: messages.QueryToRun = {
resultsPath: this.resultsPaths.resultsPath,
qlo: Uri.file(this.compiledQueryPath).toString(),
compiledUpgrade: upgradeQlo && Uri.file(upgradeQlo).toString(),
allowUnknownTemplates: true,
templateValues: this.templates,
id: callbackId,
@@ -292,7 +295,7 @@ async function checkDbschemeCompatibility(
const searchPath = getOnDiskWorkspaceFolders();
if (query.dbItem.contents !== undefined && query.dbItem.contents.dbSchemeUri !== undefined) {
const { scripts, finalDbscheme } = await cliServer.resolveUpgrades(query.dbItem.contents.dbSchemeUri.fsPath, searchPath);
const { finalDbscheme } = await cliServer.resolveUpgrades(query.dbItem.contents.dbSchemeUri.fsPath, searchPath);
const hash = async function(filename: string): Promise<string> {
return crypto.createHash('sha256').update(await fs.readFile(filename)).digest('hex');
};
@@ -311,18 +314,15 @@ async function checkDbschemeCompatibility(
const upgradableTo = await hash(finalDbscheme);
if (upgradableTo != dbschemeOfLib) {
logger.log(`Query ${query.program.queryPath} expects database scheme ${query.queryDbscheme}, but database has scheme ${query.program.dbschemePath}, and no upgrade path found`);
throw new Error(`Query ${query.program.queryPath} expects database scheme ${query.queryDbscheme}, but the current database has a different scheme, and no database upgrades are available. The current database scheme may be newer than the CodeQL query libraries in your workspace. Please try using a newer version of the query libraries.`);
reportNoUpgradePath(query);
}
if (upgradableTo == dbschemeOfLib &&
dbschemeOfDb != dbschemeOfLib) {
// Try to upgrade the database
await upgradeDatabase(
await upgradeDatabaseExplicit(
qs,
query.dbItem,
Uri.file(finalDbscheme),
getUpgradesDirectories(scripts),
progress,
token
);
@@ -330,6 +330,44 @@ async function checkDbschemeCompatibility(
}
}
function reportNoUpgradePath(query: QueryInfo) {
logger.log(`Query ${query.program.queryPath} expects database scheme ${query.queryDbscheme}, but database has scheme ${query.program.dbschemePath}, and no upgrade path found`);
throw new Error(`Query ${query.program.queryPath} expects database scheme ${query.queryDbscheme}, but the current database has a different scheme, and no database upgrades are available. The current database scheme may be newer than the CodeQL query libraries in your workspace. Please try using a newer version of the query libraries.`);
}
/**
* Compile a non-destructive upgrade.
*/
async function compileNonDestructiveUpgrade(
qs: qsClient.QueryServerClient,
upgradeTemp: tmp.DirResult,
query: QueryInfo,
progress: ProgressCallback,
token: CancellationToken,
): Promise<string> {
const searchPath = helpers.getOnDiskWorkspaceFolders();
if (query.dbItem.contents === undefined || query.dbItem.contents.dbSchemeUri === undefined) {
throw new Error('Database is invalid, and cannot be upgraded.');
}
const { scripts, matchesTarget } = await qs.cliServer.resolveUpgrades(query.dbItem.contents.dbSchemeUri.fsPath, searchPath, query.queryDbscheme);
if (!matchesTarget) {
reportNoUpgradePath(query);
}
const result = await compileDatabaseUpgradeSequence(qs, query.dbItem, query.queryDbscheme, scripts, upgradeTemp, progress, token);
if (result.compiledUpgrades === undefined) {
const error = result.error || '[no error message available]';
throw new Error(error);
}
// We can upgrade to the actual target
query.program.dbschemePath = query.queryDbscheme;
// We are new enough that we will always support single file upgrades.
return result.compiledUpgrades.compiledUpgradeFile!;
}
/**
* Prompts the user to save `document` if it has unsaved changes.
*
@@ -516,64 +554,73 @@ export async function compileAndRunQueryAgainstDatabase(
}
const query = new QueryInfo(qlProgram, db, packConfig.dbscheme, quickEvalPosition, metadata, templates);
await checkDbschemeCompatibility(cliServer, qs, query, progress, token);
let errors;
const upgradeDir = tmp.dirSync({ dir: upgradesTmpDir.name });
try {
errors = await query.compile(qs, progress, token);
} catch (e) {
if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
return createSyntheticResult(query, db, historyItemOptions, 'Query cancelled', messages.QueryResultType.CANCELLATION);
let upgradeQlo;
if (await hasNondestructiveUpgradeCapabilities(qs)) {
upgradeQlo = await compileNonDestructiveUpgrade(qs, upgradeDir, query, progress, token);
} else {
throw e;
await checkDbschemeCompatibility(cliServer, qs, query, progress, token);
}
}
if (errors.length == 0) {
const result = await query.run(qs, progress, token);
if (result.resultType !== messages.QueryResultType.SUCCESS) {
const message = result.message || 'Failed to run query';
logger.log(message);
showAndLogErrorMessage(message);
}
return {
query,
result,
database: {
name: db.name,
databaseUri: db.databaseUri.toString(true)
},
options: historyItemOptions,
logFileLocation: result.logFileLocation,
dispose: () => {
qs.logger.removeAdditionalLogLocation(result.logFileLocation);
let errors;
try {
errors = await query.compile(qs, progress, token);
} catch (e) {
if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
return createSyntheticResult(query, db, historyItemOptions, 'Query cancelled', messages.QueryResultType.CANCELLATION);
} else {
throw e;
}
};
} else {
// Error dialogs are limited in size and scrollability,
// so we include a general description of the problem,
// and direct the user to the output window for the detailed compilation messages.
// However we don't show quick eval errors there so we need to display them anyway.
qs.logger.log(`Failed to compile query ${query.program.queryPath} against database scheme ${query.program.dbschemePath}:`);
const formattedMessages: string[] = [];
for (const error of errors) {
const message = error.message || '[no error message available]';
const formatted = `ERROR: ${message} (${error.position.fileName}:${error.position.line}:${error.position.column}:${error.position.endLine}:${error.position.endColumn})`;
formattedMessages.push(formatted);
qs.logger.log(formatted);
}
if (quickEval && formattedMessages.length <= 3) {
showAndLogErrorMessage('Quick evaluation compilation failed: \n' + formattedMessages.join('\n'));
if (errors.length == 0) {
const result = await query.run(qs, upgradeQlo, progress, token);
if (result.resultType !== messages.QueryResultType.SUCCESS) {
const message = result.message || 'Failed to run query';
logger.log(message);
helpers.showAndLogErrorMessage(message);
}
return {
query,
result,
database: {
name: db.name,
databaseUri: db.databaseUri.toString(true)
},
options: historyItemOptions,
logFileLocation: result.logFileLocation,
dispose: () => {
qs.logger.removeAdditionalLogLocation(result.logFileLocation);
}
};
} else {
showAndLogErrorMessage((quickEval ? 'Quick evaluation' : 'Query') +
' compilation failed. Please make sure there are no errors in the query, the database is up to date,' +
' and the query and database use the same target language. For more details on the error, go to View > Output,' +
' and choose CodeQL Query Server from the dropdown.');
}
// Error dialogs are limited in size and scrollability,
// so we include a general description of the problem,
// and direct the user to the output window for the detailed compilation messages.
// However we don't show quick eval errors there so we need to display them anyway.
qs.logger.log(`Failed to compile query ${query.program.queryPath} against database scheme ${query.program.dbschemePath}:`);
return createSyntheticResult(query, db, historyItemOptions, 'Query had compilation errors', messages.QueryResultType.OTHER_ERROR);
const formattedMessages: string[] = [];
for (const error of errors) {
const message = error.message || '[no error message available]';
const formatted = `ERROR: ${message} (${error.position.fileName}:${error.position.line}:${error.position.column}:${error.position.endLine}:${error.position.endColumn})`;
formattedMessages.push(formatted);
qs.logger.log(formatted);
}
if (quickEval && formattedMessages.length <= 3) {
showAndLogErrorMessage('Quick evaluation compilation failed: \n' + formattedMessages.join('\n'));
} else {
showAndLogErrorMessage((quickEval ? 'Quick evaluation' : 'Query') +
' compilation failed. Please make sure there are no errors in the query, the database is up to date,' +
' and the query and database use the same target language. For more details on the error, go to View > Output,' +
' and choose CodeQL Query Server from the dropdown.');
}
return createSyntheticResult(query, db, historyItemOptions, 'Query had compilation errors', messages.QueryResultType.OTHER_ERROR);
}
} finally {
upgradeDir.removeCallback();
}
}

View File

@@ -6,6 +6,9 @@ import { logger } from './logging';
import * as messages from './pure/messages';
import * as qsClient from './queryserver-client';
import { upgradesTmpDir } from './run-queries';
import * as tmp from 'tmp';
import * as path from 'path';
import { getOnDiskWorkspaceFolders } from './helpers';
/**
* Maximum number of lines to include from database upgrade message,
@@ -15,81 +18,97 @@ import { upgradesTmpDir } from './run-queries';
const MAX_UPGRADE_MESSAGE_LINES = 10;
/**
* Checks whether the given database can be upgraded to the given target DB scheme,
* and whether the user wants to proceed with the upgrade.
* Reports errors to both the user and the console.
* @returns the `UpgradeParams` needed to start the upgrade, if the upgrade is possible and was confirmed by the user, or `undefined` otherwise.
* Check that we support non-destructive upgrades.
*
* This requires 3 features. The ability to compile an upgrade sequence; The ability to
* run a non-desturcitve upgrades as a query; the ability to specify a target when
* resolving upgrades.
*/
async function checkAndConfirmDatabaseUpgrade(
qs: qsClient.QueryServerClient,
export async function hasNondestructiveUpgradeCapabilities(qs: qsClient.QueryServerClient): Promise<boolean> {
// TODO change to actual version when known
// Note it is probably something 2.4.something
return (await qs.cliServer.getVersion()).compare('2.3.2') >= 0;
}
/**
* Compile a database upgrade sequence.
* Callers must check that this is valid with the current queryserver first.
*/
export async function compileDatabaseUpgradeSequence(qs: qsClient.QueryServerClient,
db: DatabaseItem,
targetDbScheme: vscode.Uri,
upgradesDirectories: vscode.Uri[],
targetDbScheme: string,
resolvedSequence: string[],
currentUpgradeTmp: tmp.DirResult,
progress: ProgressCallback,
token: vscode.CancellationToken,
): Promise<messages.UpgradeParams | undefined> {
token: vscode.CancellationToken): Promise<messages.SingleFileCompiledUpgradeResult> {
if (db.contents === undefined || db.contents.dbSchemeUri === undefined) {
throw new Error('Database is invalid, and cannot be upgraded.');
}
const params: messages.UpgradeParams = {
fromDbscheme: db.contents.dbSchemeUri.fsPath,
toDbscheme: targetDbScheme.fsPath,
additionalUpgrades: upgradesDirectories.map(uri => uri.fsPath)
};
// If possible just compile the upgrade sequence
return await qs.sendRequest(messages.compileUpgradeSequence, {
upgradeTempDir: currentUpgradeTmp.name,
finalDbscheme: targetDbScheme,
initialDbscheme: db.contents.dbSchemeUri.fsPath,
upgradePaths: resolvedSequence
}, token, progress);
}
let checkUpgradeResult: messages.CheckUpgradeResult;
try {
qs.logger.log('Checking database upgrade...');
checkUpgradeResult = await checkDatabaseUpgrade(qs, params, progress, token);
}
catch (e) {
throw new Error(`Database cannot be upgraded: ${e}`);
}
finally {
qs.logger.log('Done checking database upgrade.');
}
const checkedUpgrades = checkUpgradeResult.checkedUpgrades;
if (checkedUpgrades === undefined) {
const error = checkUpgradeResult.upgradeError || '[no error message available]';
throw new Error(`Database cannot be upgraded: ${error}`);
}
if (checkedUpgrades.scripts.length === 0) {
async function compileDatabaseUpgrade(
qs: qsClient.QueryServerClient,
db: DatabaseItem,
targetDbScheme: string,
resolvedSequence: string[],
currentUpgradeTmp: tmp.DirResult,
progress: ProgressCallback,
token: vscode.CancellationToken
): Promise<messages.CompileUpgradeResult> {
if (await hasNondestructiveUpgradeCapabilities(qs)) {
return await compileDatabaseUpgradeSequence(qs, db, targetDbScheme, resolvedSequence, currentUpgradeTmp, progress, token);
} else {
if (db.contents === undefined || db.contents.dbSchemeUri === undefined) {
throw new Error('Database is invalid, and cannot be upgraded.');
}
// We have the upgrades we want but compileUpgrade
// requires searching for them. So we use the parent directories of the upgrades
// as the uograde path.
const parentDirs = resolvedSequence.map(dir => path.dirname(dir));
const uniqueParentDirs = new Set(parentDirs);
progress({
step: 3,
step: 1,
maxStep: 3,
message: 'Database is already up to date; nothing to do.'
message: 'Checking for database upgrades'
});
return;
return qs.sendRequest(messages.compileUpgrade, {
upgrade: {
fromDbscheme: db.contents.dbSchemeUri.fsPath,
toDbscheme: targetDbScheme,
additionalUpgrades: Array.from(uniqueParentDirs)
},
upgradeTempDir: currentUpgradeTmp.name,
singleFileUpgrades: true,
}, token, progress);
}
}
/**
* Checks whether the user wants to proceed with the upgrade.
* Reports errors to both the user and the console.
*/
async function checkAndConfirmDatabaseUpgrade(
compiled: messages.CompiledUpgrades,
db: DatabaseItem,
): Promise<void> {
let curSha = checkedUpgrades.initialSha;
let descriptionMessage = '';
for (const script of checkedUpgrades.scripts) {
const descriptions = getUpgradeDescriptions(compiled);
for (const script of descriptions) {
descriptionMessage += `Would perform upgrade: ${script.description}\n`;
descriptionMessage += `\t-> Compatibility: ${script.compatibility}\n`;
curSha = script.newSha;
}
const targetSha = checkedUpgrades.targetSha;
if (curSha != targetSha) {
// Newlines aren't rendered in notifications: https://github.com/microsoft/vscode/issues/48900
// A modal dialog would be rendered better, but is more intrusive.
await showAndLogErrorMessage(`Database cannot be upgraded to the target database scheme.
Can upgrade from ${checkedUpgrades.initialSha} (current) to ${curSha}, but cannot reach ${targetSha} (target).`);
// TODO: give a more informative message if we think the DB is ahead of the target DB scheme
return;
}
logger.log(descriptionMessage);
// If the quiet flag is set, do the upgrade without a popup.
if (qs.cliServer.quiet) {
return params;
}
// Ask the user to confirm the upgrade.
const showLogItem: vscode.MessageItem = { title: 'No, Show Changes', isCloseAffordance: true };
@@ -104,113 +123,101 @@ async function checkAndConfirmDatabaseUpgrade(
dialogOptions.push(showLogItem);
}
const message = `Should the database ${db.databaseUri.fsPath} be upgraded?\n\n${messageLines.join('\n')}`;
const message = `Should the database ${db} be upgraded?\n\n${messageLines.join('\n')}`;
const chosenItem = await vscode.window.showInformationMessage(message, { modal: true }, ...dialogOptions);
if (chosenItem === showLogItem) {
logger.outputChannel.show();
}
if (chosenItem === yesItem) {
return params;
}
else {
if (chosenItem !== yesItem) {
throw new UserCancellationException('User cancelled the database upgrade.');
}
}
/**
* Get the descriptions from a compiled upgrade
*/
function getUpgradeDescriptions(compiled: messages.CompiledUpgrades): messages.UpgradeDescription[] {
// We use the presence of compiledUpgradeFile to check
// if it is multifile or not. We need to explicitly check undefined
// as the types claim the empty string is a valid value
if (compiled.compiledUpgradeFile === undefined) {
return compiled.scripts.map(script => script.description);
} else {
return compiled.descriptions;
}
}
/**
* Command handler for 'Upgrade Database'.
* Attempts to upgrade the given database to the given target DB scheme, using the given directory of upgrades.
* First performs a dry-run and prompts the user to confirm the upgrade.
* Reports errors during compilation and evaluation of upgrades to the user.
*/
export async function upgradeDatabase(
export async function upgradeDatabaseExplicit(
qs: qsClient.QueryServerClient,
db: DatabaseItem, targetDbScheme: vscode.Uri,
upgradesDirectories: vscode.Uri[],
db: DatabaseItem,
progress: ProgressCallback,
token: vscode.CancellationToken,
): Promise<messages.RunUpgradeResult | undefined> {
const upgradeParams = await checkAndConfirmDatabaseUpgrade(qs, db, targetDbScheme, upgradesDirectories, progress, token);
if (upgradeParams === undefined) {
return;
const searchPath: string[] = getOnDiskWorkspaceFolders();
if (db.contents === undefined || db.contents.dbSchemeUri === undefined) {
throw new Error('Database is invalid, and cannot be upgraded.');
}
const upgradeInfo = await qs.cliServer.resolveUpgrades(
db.contents.dbSchemeUri.fsPath,
searchPath
);
let compileUpgradeResult: messages.CompileUpgradeResult;
const { scripts, finalDbscheme } = upgradeInfo;
if (finalDbscheme === undefined) {
throw new Error('Could not determine target dbscheme to upgrade to.');
}
const currentUpgradeTmp = tmp.dirSync({ dir: upgradesTmpDir.name, prefix: 'upgrade_', keep: false, unsafeCleanup: true });
try {
compileUpgradeResult = await compileDatabaseUpgrade(qs, upgradeParams, progress, token);
}
catch (e) {
showAndLogErrorMessage(`Compilation of database upgrades failed: ${e}`);
return;
}
finally {
qs.logger.log('Done compiling database upgrade.');
}
if (compileUpgradeResult.compiledUpgrades === undefined) {
const error = compileUpgradeResult.error || '[no error message available]';
showAndLogErrorMessage(`Compilation of database upgrades failed: ${error}`);
return;
}
try {
qs.logger.log('Running the following database upgrade:');
// We use the presence of compiledUpgradeFile to check
// if it is multifile or not. We need to explicitly check undefined
// as the types claim the empty string is a valid value
if (compileUpgradeResult.compiledUpgrades.compiledUpgradeFile === undefined) {
qs.logger.log(compileUpgradeResult.compiledUpgrades.scripts.map(s => s.description.description).join('\n'));
} else {
qs.logger.log(compileUpgradeResult.compiledUpgrades.descriptions.map(s => s.description).join('\n'));
let compileUpgradeResult: messages.CompileUpgradeResult;
try {
compileUpgradeResult = await compileDatabaseUpgrade(qs, db, finalDbscheme, scripts, currentUpgradeTmp, progress, token);
}
return await runDatabaseUpgrade(qs, db, compileUpgradeResult.compiledUpgrades, progress, token);
catch (e) {
showAndLogErrorMessage(`Compilation of database upgrades failed: ${e}`);
return;
}
finally {
qs.logger.log('Done compiling database upgrade.');
}
if (!compileUpgradeResult.compiledUpgrades) {
const error = compileUpgradeResult.error || '[no error message available]';
showAndLogErrorMessage(`Compilation of database upgrades failed: ${error}`);
return;
}
// If the quiet flag is set, do the upgrade without a popup.
if (!qs.cliServer.quiet) {
await checkAndConfirmDatabaseUpgrade(compileUpgradeResult.compiledUpgrades, db);
}
try {
qs.logger.log('Running the following database upgrade:');
getUpgradeDescriptions(compileUpgradeResult.compiledUpgrades).map(s => s.description).join('\n');
return await runDatabaseUpgrade(qs, db, compileUpgradeResult.compiledUpgrades, progress, token);
}
catch (e) {
showAndLogErrorMessage(`Database upgrade failed: ${e}`);
return;
} finally {
qs.logger.log('Done running database upgrade.');
}
} finally {
currentUpgradeTmp.removeCallback();
}
catch (e) {
showAndLogErrorMessage(`Database upgrade failed: ${e}`);
return;
}
finally {
qs.logger.log('Done running database upgrade.');
}
}
async function checkDatabaseUpgrade(
qs: qsClient.QueryServerClient,
upgradeParams: messages.UpgradeParams,
progress: ProgressCallback,
token: vscode.CancellationToken,
): Promise<messages.CheckUpgradeResult> {
progress({
step: 1,
maxStep: 3,
message: 'Checking for database upgrades'
});
return qs.sendRequest(messages.checkUpgrade, upgradeParams, token, progress);
}
async function compileDatabaseUpgrade(
qs: qsClient.QueryServerClient,
upgradeParams: messages.UpgradeParams,
progress: ProgressCallback,
token: vscode.CancellationToken,
): Promise<messages.CompileUpgradeResult> {
const params: messages.CompileUpgradeParams = {
upgrade: upgradeParams,
upgradeTempDir: upgradesTmpDir.name,
singleFileUpgrades: true
};
progress({
step: 2,
maxStep: 3,
message: 'Compiling database upgrades'
});
return qs.sendRequest(messages.compileUpgrade, params, token, progress);
}
async function runDatabaseUpgrade(