mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Add source to database item

Browse Source 
This adds a source property to the database item to store the source of
the database, for example GitHub or an internet URL. This will be used
to automatically check for updates to GitHub-downloaded databases in the
future.
This commit is contained in:
Koen Vlaswinkel
2023-11-13 11:13:36 +01:00
parent bc2847a12e
commit bca7ecb782
20 changed files with 206 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
extensions/ql-vscode/src/databases/config/db-config.ts
View File

@@ -1,6 +1,8 @@
// Contains models and consts for the data we want to store in the database config.
// Changes to these models should be done carefully and account for backwards compatibility of data.
import { DatabaseSource } from "../local-databases/database-source";
export const DB_CONFIG_VERSION = 1;
export interface DbConfig {
@@ -88,6 +90,7 @@ export interface LocalDatabase {
name: string;
dateAdded: number;
language: string;
source: DatabaseSource;
storagePath: string;
}

28
extensions/ql-vscode/src/databases/database-fetcher.ts
View File

@@ -33,6 +33,7 @@ import { allowHttp } from "../config";
import { showAndLogInformationMessage } from "../common/logging";
import { AppOctokit } from "../common/octokit";
import { getLanguageDisplayName } from "../common/query-language";
import { DatabaseSource } from "./local-databases/database-source";
/**
* Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file.
@@ -62,6 +63,10 @@ export async function promptImportInternetDatabase(
databaseManager,
storagePath,
undefined,
{
type: "url",
url: databaseUrl,
},
progress,
cli,
);
@@ -199,7 +204,7 @@ export async function downloadGitHubDatabase(
return;
}
const { databaseUrl, name, owner } = result;
const { databaseUrl, name, owner, commitOid } = result;
/**
* The 'token' property of the token object returned by `octokit.auth()`.
@@ -221,6 +226,11 @@ export async function downloadGitHubDatabase(
databaseManager,
storagePath,
`${owner}/${name}`,
{
type: "github",
repository: nwo,
commitOid,
},
progress,
cli,
makeSelected,
@@ -250,6 +260,10 @@ export async function importArchiveDatabase(
databaseManager,
storagePath,
undefined,
{
type: "archive",
path: databaseUrl,
},
progress,
cli,
);
@@ -282,6 +296,7 @@ export async function importArchiveDatabase(
* @param databaseManager the DatabaseManager
* @param storagePath where to store the unzipped database.
* @param nameOverride a name for the database that overrides the default
* @param source the source of the database
* @param progress callback to send progress messages to
* @param makeSelected make the new database selected in the databases panel (default: true)
* @param addSourceArchiveFolder whether to add a workspace folder containing the source archive to the workspace
@@ -292,6 +307,7 @@ async function databaseArchiveFetcher(
databaseManager: DatabaseManager,
storagePath: string,
nameOverride: string | undefined,
source: DatabaseSource,
progress: ProgressCallback,
cli?: CodeQLCliServer,
makeSelected = true,
@@ -336,6 +352,7 @@ async function databaseArchiveFetcher(
const item = await databaseManager.openDatabase(
Uri.file(dbPath),
source,
makeSelected,
nameOverride,
{
@@ -533,6 +550,7 @@ export async function convertGithubNwoToDatabaseUrl(
databaseUrl: string;
owner: string;
name: string;
commitOid: string | null;
}
| undefined
> {
@@ -553,10 +571,18 @@ export async function convertGithubNwoToDatabaseUrl(
}
}
const databaseForLanguage = response.data.find(
(db: any) => db.language === language,
);
if (!databaseForLanguage) {
throw new Error(`No database found for language '${language}'`);
}
return {
databaseUrl: `https://api.github.com/repos/${owner}/${repo}/code-scanning/codeql/databases/${language}`,
owner,
name: repo,
commitOid: databaseForLanguage.commit_oid,
};
} catch (e) {
void extLogger.log(`Error: ${getErrorMessage(e)}`);

3
extensions/ql-vscode/src/databases/db-item.ts
View File

@@ -1,5 +1,7 @@
// This file contains models that are used to represent the databases.
import { DatabaseSource } from "./local-databases/database-source";
export enum DbItemKind {
RootLocal = "RootLocal",
LocalList = "LocalList",
@@ -38,6 +40,7 @@ export interface LocalDatabaseDbItem {
databaseName: string;
dateAdded: number;
language: string;
source: DatabaseSource;
storagePath: string;
parentListName?: string;
}

1
extensions/ql-vscode/src/databases/db-tree-creator.ts
View File

@@ -197,6 +197,7 @@ function createLocalDb(
databaseName: db.name,
dateAdded: db.dateAdded,
language: db.language,
source: db.source,
storagePath: db.storagePath,
selected: !!selected,
parentListName: listName,

11
extensions/ql-vscode/src/databases/local-databases-ui.ts
View File

@@ -367,6 +367,9 @@ export class DatabaseUI extends DisposableObject {
await this.databaseManager.openDatabase(
uri,
{
type: "folder",
},
makeSelected,
nameOverride,
{
@@ -704,7 +707,9 @@ export class DatabaseUI extends DisposableObject {
this.queryServer?.cliServer,
);
} else {
await this.databaseManager.openDatabase(uri);
await this.databaseManager.openDatabase(uri, {
type: "folder",
});
}
} catch (e) {
// rethrow and let this be handled by default error handling.
@@ -819,7 +824,9 @@ export class DatabaseUI extends DisposableObject {
if (byFolder) {
const fixedUri = await this.fixDbUri(uri);
// we are selecting a database folder
return await this.databaseManager.openDatabase(fixedUri);
return await this.databaseManager.openDatabase(fixedUri, {
type: "folder",
});
} else {
// we are selecting a database archive. Must unzip into a workspace-controlled area
// before importing.

5
extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts
View File

@@ -14,6 +14,7 @@ import { isLikelyDatabaseRoot } from "./db-contents-heuristics";
import { stat } from "fs-extra";
import { containsPath, pathsEqual } from "../../common/files";
import { DatabaseContents } from "./database-contents";
import { DatabaseSource } from "./database-source";
export class DatabaseItemImpl implements DatabaseItem {
// These are only public in the implementation, they are readonly in the interface
@@ -61,6 +62,10 @@ export class DatabaseItemImpl implements DatabaseItem {
return this.options.dateAdded;
}
public get source(): DatabaseSource | undefined {
return this.options.source;
}
public resolveSourceFile(uriStr: string | undefined): vscode.Uri {
const sourceArchive = this.sourceArchive;
const uri = uriStr ? vscode.Uri.parse(uriStr, true) : undefined;

6
extensions/ql-vscode/src/databases/local-databases/database-item.ts
View File

@@ -2,6 +2,7 @@ import vscode from "vscode";
import * as cli from "../../codeql-cli/cli";
import { DatabaseContents } from "./database-contents";
import { DatabaseOptions } from "./database-options";
import { DatabaseSource } from "./database-source";
/** An item in the list of available databases */
export interface DatabaseItem {
@@ -25,6 +26,11 @@ export interface DatabaseItem {
*/
readonly dateAdded: number | undefined;
/**
* The source this database item was added from or undefined if unknown.
*/
readonly source: DatabaseSource | undefined;
/** If the database is invalid, describes why. */
readonly error: Error | undefined;

16
extensions/ql-vscode/src/databases/local-databases/database-manager.ts
View File

@@ -34,6 +34,7 @@ import { DatabaseChangedEvent, DatabaseEventKind } from "./database-events";
import { DatabaseResolver } from "./database-resolver";
import { telemetryListener } from "../../common/vscode/telemetry";
import { LanguageContextStore } from "../../language-context-store";
import { DatabaseSource } from "./database-source";
/**
* The name of the key in the workspaceState dictionary in which we
@@ -131,6 +132,7 @@ export class DatabaseManager extends DisposableObject {
*/
public async openDatabase(
uri: vscode.Uri,
source: DatabaseSource | undefined,
makeSelected = true,
displayName?: string,
{
@@ -138,7 +140,11 @@ export class DatabaseManager extends DisposableObject {
addSourceArchiveFolder = true,
}: OpenDatabaseOptions = {},
): Promise<DatabaseItem> {
const databaseItem = await this.createDatabaseItem(uri, displayName);
const databaseItem = await this.createDatabaseItem(
uri,
source,
displayName,
);
return await this.addExistingDatabaseItem(
databaseItem,
@@ -189,6 +195,7 @@ export class DatabaseManager extends DisposableObject {
*/
private async createDatabaseItem(
uri: vscode.Uri,
source: DatabaseSource | undefined,
displayName: string | undefined,
): Promise<DatabaseItemImpl> {
const contents = await DatabaseResolver.resolveDatabaseContents(uri);
@@ -197,6 +204,7 @@ export class DatabaseManager extends DisposableObject {
displayName,
dateAdded: Date.now(),
language: await this.getPrimaryLanguage(uri.fsPath),
source,
};
const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions);
@@ -212,6 +220,7 @@ export class DatabaseManager extends DisposableObject {
*/
public async createOrOpenDatabaseItem(
uri: vscode.Uri,
source: DatabaseSource | undefined,
): Promise<DatabaseItem> {
const existingItem = this.findDatabaseItem(uri);
if (existingItem !== undefined) {
@@ -220,7 +229,7 @@ export class DatabaseManager extends DisposableObject {
}
// We don't add this to the list automatically, but the user can add it later.
return this.createDatabaseItem(uri, undefined);
return this.createDatabaseItem(uri, source, undefined);
}
public async createSkeletonPacks(databaseItem: DatabaseItem) {
@@ -355,6 +364,7 @@ export class DatabaseManager extends DisposableObject {
let displayName: string | undefined = undefined;
let dateAdded = undefined;
let language = undefined;
let source = undefined;
if (state.options) {
if (typeof state.options.displayName === "string") {
displayName = state.options.displayName;
@@ -363,6 +373,7 @@ export class DatabaseManager extends DisposableObject {
dateAdded = state.options.dateAdded;
}
language = state.options.language;
source = state.options.source;
}
const dbBaseUri = vscode.Uri.parse(state.uri, true);
@@ -375,6 +386,7 @@ export class DatabaseManager extends DisposableObject {
displayName,
dateAdded,
language,
source,
};
const item = new DatabaseItemImpl(dbBaseUri, undefined, fullOptions);

4
extensions/ql-vscode/src/databases/local-databases/database-options.ts
View File

@@ -1,10 +1,14 @@
import { DatabaseSource } from "./database-source";
export interface DatabaseOptions {
displayName?: string;
dateAdded?: number | undefined;
language?: string;
source?: DatabaseSource;
}
export interface FullDatabaseOptions extends DatabaseOptions {
dateAdded: number | undefined;
language: string | undefined;
source: DatabaseSource | undefined;
}

30
extensions/ql-vscode/src/databases/local-databases/database-source.ts Normal file
View File

@@ -0,0 +1,30 @@
interface DatabaseSourceFolder {
type: "folder";
}
interface DatabaseSourceArchive {
type: "archive";
path: string;
}
interface DatabaseSourceGitHub {
type: "github";
repository: string;
commitOid: string | null;
}
interface DatabaseSourceInternet {
type: "url";
url: string;
}
interface DatabaseSourceDebugger {
type: "debugger";
}
export type DatabaseSource =
| DatabaseSourceFolder
| DatabaseSourceArchive
| DatabaseSourceGitHub
| DatabaseSourceInternet
| DatabaseSourceDebugger;

4
extensions/ql-vscode/src/debugger/debugger-ui.ts
View File

@@ -105,7 +105,9 @@ class QLDebugAdapterTracker
body: CodeQLProtocol.EvaluationStartedEvent["body"],
): Promise<void> {
const dbUri = Uri.file(this.configuration.database);
const dbItem = await this.dbm.createOrOpenDatabaseItem(dbUri);
const dbItem = await this.dbm.createOrOpenDatabaseItem(dbUri, {
type: "debugger",
});
// When cancellation is requested from the query history view, we just stop the debug session.
const tokenSource = new CancellationTokenSource();

1
extensions/ql-vscode/src/query-testing/test-runner.ts
View File

@@ -103,6 +103,7 @@ export class TestRunner extends DisposableObject {
try {
const reopenedDatabase = await this.databaseManager.openDatabase(
uri,
closedDatabase.source,
false,
);
await this.databaseManager.renameDatabaseItem(

3
extensions/ql-vscode/test/factories/databases/databases.ts
View File

@@ -11,6 +11,9 @@ export function mockDbOptions(): FullDatabaseOptions {
return {
dateAdded: 123,
language: "",
source: {
type: "folder",
},
};
}

6
extensions/ql-vscode/test/factories/db-config-factories.ts
View File

@@ -7,6 +7,7 @@ import {
SelectedDbItem,
DB_CONFIG_VERSION,
} from "../../src/databases/config/db-config";
import { DatabaseSource } from "../../src/databases/local-databases/database-source";
export function createDbConfig({
remoteLists = [],
@@ -45,16 +46,21 @@ export function createLocalDbConfigItem({
dateAdded = faker.date.past().getTime(),
language = `language${faker.number.int()}`,
storagePath = `storagePath${faker.number.int()}`,
source = {
type: "folder",
},
}: {
name?: string;
dateAdded?: number;
language?: string;
storagePath?: string;
source?: DatabaseSource;
} = {}): LocalDatabase {
return {
name,
dateAdded,
language,
storagePath,
source,
};
}

6
extensions/ql-vscode/test/factories/db-item-factories.ts
View File

@@ -12,6 +12,7 @@ import {
RootLocalDbItem,
RootRemoteDbItem,
} from "../../src/databases/db-item";
import { DatabaseSource } from "../../src/databases/local-databases/database-source";
// Root Remote Db Items
export function createRootRemoteDbItem({
@@ -124,12 +125,16 @@ export function createLocalDatabaseDbItem({
language = `language${faker.number.int()}`,
storagePath = `storagePath${faker.number.int()}`,
selected = false,
source = {
type: "folder",
},
}: {
databaseName?: string;
dateAdded?: number;
language?: string;
storagePath?: string;
selected?: boolean;
source?: DatabaseSource;
} = {}): LocalDatabaseDbItem {
return {
kind: DbItemKind.LocalDatabase,
@@ -138,6 +143,7 @@ export function createLocalDatabaseDbItem({
dateAdded,
language,
storagePath,
source,
};
}

3
extensions/ql-vscode/test/unit-tests/databases/db-item-selection.test.ts
View File

@@ -57,6 +57,9 @@ describe("db item selection", () => {
dateAdded: 1234,
language: "javascript",
storagePath: "/foo/bar",
source: {
type: "folder",
},
selected: true,
});
});

19
extensions/ql-vscode/test/unit-tests/databases/db-tree-creator.test.ts
View File

@@ -337,12 +337,18 @@ describe("db tree creator", () => {
dateAdded: 1668428293677,
language: QueryLanguage.Cpp,
storagePath: "/path/to/db1/",
source: {
type: "folder",
},
},
{
name: "db2",
dateAdded: 1668428472731,
language: "cpp",
storagePath: "/path/to/db2/",
source: {
type: "folder",
},
},
],
},
@@ -354,6 +360,9 @@ describe("db tree creator", () => {
dateAdded: 1668428472731,
language: "ruby",
storagePath: "/path/to/db3/",
source: {
type: "folder",
},
},
],
},
@@ -380,6 +389,7 @@ describe("db tree creator", () => {
databaseName: db.name,
dateAdded: db.dateAdded,
language: db.language,
source: db.source,
storagePath: db.storagePath,
parentListName: dbConfig.databases.local.lists[0].name,
})),
@@ -395,6 +405,7 @@ describe("db tree creator", () => {
databaseName: db.name,
dateAdded: db.dateAdded,
language: db.language,
source: db.source,
storagePath: db.storagePath,
parentListName: dbConfig.databases.local.lists[1].name,
})),
@@ -409,12 +420,18 @@ describe("db tree creator", () => {
dateAdded: 1668428293677,
language: "csharp",
storagePath: "/path/to/db1/",
source: {
type: "folder",
},
},
{
name: "db2",
dateAdded: 1668428472731,
language: "go",
storagePath: "/path/to/db2/",
source: {
type: "folder",
},
},
],
});
@@ -434,6 +451,7 @@ describe("db tree creator", () => {
databaseName: dbConfig.databases.local.databases[0].name,
dateAdded: dbConfig.databases.local.databases[0].dateAdded,
language: dbConfig.databases.local.databases[0].language,
source: dbConfig.databases.local.databases[0].source,
storagePath: dbConfig.databases.local.databases[0].storagePath,
});
expect(localDatabaseNodes[1]).toEqual({
@@ -442,6 +460,7 @@ describe("db tree creator", () => {
databaseName: dbConfig.databases.local.databases[1].name,
dateAdded: dbConfig.databases.local.databases[1].dateAdded,
language: dbConfig.databases.local.databases[1].language,
source: dbConfig.databases.local.databases[1].source,
storagePath: dbConfig.databases.local.databases[1].storagePath,
});
});

30
extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts
View File

@@ -186,12 +186,18 @@ describe("db panel rendering nodes", () => {
dateAdded: 1668428293677,
language: QueryLanguage.Cpp,
storagePath: "/path/to/db1/",
source: {
type: "folder",
},
},
{
name: "db2",
dateAdded: 1668428472731,
language: QueryLanguage.Cpp,
storagePath: "/path/to/db2/",
source: {
type: "folder",
},
},
],
},
@@ -203,6 +209,9 @@ describe("db panel rendering nodes", () => {
dateAdded: 1668428472731,
language: "ruby",
storagePath: "/path/to/db3/",
source: {
type: "folder",
},
},
],
},
@@ -238,6 +247,9 @@ describe("db panel rendering nodes", () => {
language: QueryLanguage.Cpp,
storagePath: "/path/to/db1/",
selected: false,
source: {
type: "folder",
},
},
{
kind: DbItemKind.LocalDatabase,
@@ -246,6 +258,9 @@ describe("db panel rendering nodes", () => {
language: QueryLanguage.Cpp,
storagePath: "/path/to/db2/",
selected: false,
source: {
type: "folder",
},
},
]);
checkLocalListItem(localListItems[1], "my-list-2", [
@@ -256,6 +271,9 @@ describe("db panel rendering nodes", () => {
language: "ruby",
storagePath: "/path/to/db3/",
selected: false,
source: {
type: "folder",
},
},
]);
});
@@ -268,12 +286,18 @@ describe("db panel rendering nodes", () => {
dateAdded: 1668428293677,
language: "csharp",
storagePath: "/path/to/db1/",
source: {
type: "folder",
},
},
{
name: "db2",
dateAdded: 1668428472731,
language: "go",
storagePath: "/path/to/db2/",
source: {
type: "folder",
},
},
],
});
@@ -306,6 +330,9 @@ describe("db panel rendering nodes", () => {
language: "csharp",
storagePath: "/path/to/db1/",
selected: false,
source: {
type: "folder",
},
});
checkLocalDatabaseItem(localDatabaseItems[1], {
kind: DbItemKind.LocalDatabase,
@@ -314,6 +341,9 @@ describe("db panel rendering nodes", () => {
language: "go",
storagePath: "/path/to/db2/",
selected: false,
source: {
type: "folder",
},
});
});
});

29
extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-queries/local-databases.test.ts
View File

@@ -597,6 +597,9 @@ describe("local databases", () => {
const options: FullDatabaseOptions = {
dateAdded: 123,
language,
source: {
type: "folder",
},
};
mockDbItem = createMockDB(dir, options);
@@ -718,19 +721,28 @@ describe("local databases", () => {
});
it("should resolve the database contents", async () => {
await databaseManager.openDatabase(mockDbItem.databaseUri);
await databaseManager.openDatabase(
mockDbItem.databaseUri,
mockDbItem.source,
);
expect(resolveDatabaseContentsSpy).toBeCalledTimes(2);
});
it("should set the database as the currently selected one", async () => {
await databaseManager.openDatabase(mockDbItem.databaseUri);
await databaseManager.openDatabase(
mockDbItem.databaseUri,
mockDbItem.source,
);
expect(setCurrentDatabaseItemSpy).toBeCalledTimes(1);
});
it("should add database source archive folder", async () => {
await databaseManager.openDatabase(mockDbItem.databaseUri);
await databaseManager.openDatabase(
mockDbItem.databaseUri,
mockDbItem.source,
);
expect(addDatabaseSourceArchiveFolderSpy).toBeCalledTimes(1);
});
@@ -746,6 +758,7 @@ describe("local databases", () => {
await databaseManager.openDatabase(
mockDbItem.databaseUri,
mockDbItem.source,
makeSelected,
nameOverride,
{ isTutorialDatabase },
@@ -759,7 +772,10 @@ describe("local databases", () => {
it("should create a skeleton QL pack", async () => {
jest.spyOn(config, "isCodespacesTemplate").mockReturnValue(true);
await databaseManager.openDatabase(mockDbItem.databaseUri);
await databaseManager.openDatabase(
mockDbItem.databaseUri,
mockDbItem.source,
);
expect(createSkeletonPacksSpy).toBeCalledTimes(1);
});
@@ -770,7 +786,10 @@ describe("local databases", () => {
it("should not create a skeleton QL pack", async () => {
jest.spyOn(config, "isCodespacesTemplate").mockReturnValue(false);
await databaseManager.openDatabase(mockDbItem.databaseUri);
await databaseManager.openDatabase(
mockDbItem.databaseUri,
mockDbItem.source,
);
expect(createSkeletonPacksSpy).toBeCalledTimes(0);
});
});

11
extensions/ql-vscode/test/vscode-tests/no-workspace/query-testing/test-runner.test.ts
View File

@@ -39,12 +39,18 @@ describe("test-runner", () => {
const preTestDatabaseItem = new DatabaseItemImpl(
Uri.file("/path/to/test/dir/dir.testproj"),
undefined,
mockedObject<FullDatabaseOptions>({ displayName: "custom display name" }),
mockedObject<FullDatabaseOptions>({
displayName: "custom display name",
source: { type: "folder" },
}),
);
const postTestDatabaseItem = new DatabaseItemImpl(
Uri.file("/path/to/test/dir/dir.testproj"),
undefined,
mockedObject<FullDatabaseOptions>({ displayName: "default name" }),
mockedObject<FullDatabaseOptions>({
displayName: "default name",
source: { type: "folder" },
}),
);
beforeEach(() => {
@@ -160,6 +166,7 @@ describe("test-runner", () => {
expect(openDatabaseSpy).toBeCalledTimes(1);
expect(openDatabaseSpy).toBeCalledWith(
preTestDatabaseItem.databaseUri,
preTestDatabaseItem.source,
false,
);