mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #3066 from github/koesie10/add-database-source

Browse Source 
Add origin metadata to database items
This commit is contained in:
Koen Vlaswinkel
2023-11-15 16:12:32 +01:00
committed by GitHub
parent a271f7b36e 7fe707a42d
commit 6fc567b5b9
20 changed files with 219 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
extensions/ql-vscode/src/databases/config/db-config.ts
View File

@@ -1,6 +1,8 @@
// Contains models and consts for the data we want to store in the database config.
// Changes to these models should be done carefully and account for backwards compatibility of data.
import { DatabaseOrigin } from "../local-databases/database-origin";
export const DB_CONFIG_VERSION = 1;
export interface DbConfig {
@@ -88,6 +90,7 @@ export interface LocalDatabase {
name: string;
dateAdded: number;
language: string;
origin: DatabaseOrigin;
storagePath: string;
}

35
extensions/ql-vscode/src/databases/database-fetcher.ts
View File

@@ -33,6 +33,7 @@ import { addDatabaseSourceToWorkspace, allowHttp } from "../config";
import { showAndLogInformationMessage } from "../common/logging";
import { AppOctokit } from "../common/octokit";
import { getLanguageDisplayName } from "../common/query-language";
import { DatabaseOrigin } from "./local-databases/database-origin";
/**
* Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file.
@@ -62,6 +63,10 @@ export async function promptImportInternetDatabase(
databaseManager,
storagePath,
undefined,
{
type: "url",
url: databaseUrl,
},
progress,
cli,
);
@@ -199,7 +204,8 @@ export async function downloadGitHubDatabase(
return;
}
const { databaseUrl, name, owner } = result;
const { databaseUrl, name, owner, databaseId, databaseCreatedAt, commitOid } =
result;
/**
* The 'token' property of the token object returned by `octokit.auth()`.
@@ -221,6 +227,13 @@ export async function downloadGitHubDatabase(
databaseManager,
storagePath,
`${owner}/${name}`,
{
type: "github",
repository: nwo,
databaseId,
databaseCreatedAt,
commitOid,
},
progress,
cli,
makeSelected,
@@ -250,6 +263,10 @@ export async function importArchiveDatabase(
databaseManager,
storagePath,
undefined,
{
type: "archive",
path: databaseUrl,
},
progress,
cli,
);
@@ -282,6 +299,7 @@ export async function importArchiveDatabase(
* @param databaseManager the DatabaseManager
* @param storagePath where to store the unzipped database.
* @param nameOverride a name for the database that overrides the default
* @param origin the origin of the database
* @param progress callback to send progress messages to
* @param makeSelected make the new database selected in the databases panel (default: true)
* @param addSourceArchiveFolder whether to add a workspace folder containing the source archive to the workspace
@@ -292,6 +310,7 @@ async function databaseArchiveFetcher(
databaseManager: DatabaseManager,
storagePath: string,
nameOverride: string | undefined,
origin: DatabaseOrigin,
progress: ProgressCallback,
cli?: CodeQLCliServer,
makeSelected = true,
@@ -336,6 +355,7 @@ async function databaseArchiveFetcher(
const item = await databaseManager.openDatabase(
Uri.file(dbPath),
origin,
makeSelected,
nameOverride,
{
@@ -533,6 +553,9 @@ export async function convertGithubNwoToDatabaseUrl(
databaseUrl: string;
owner: string;
name: string;
databaseId: number;
databaseCreatedAt: string;
commitOid: string | null;
}
| undefined
> {
@@ -553,10 +576,20 @@ export async function convertGithubNwoToDatabaseUrl(
}
}
const databaseForLanguage = response.data.find(
(db: any) => db.language === language,
);
if (!databaseForLanguage) {
throw new Error(`No database found for language '${language}'`);
}
return {
databaseUrl: `https://api.github.com/repos/${owner}/${repo}/code-scanning/codeql/databases/${language}`,
owner,
name: repo,
databaseId: databaseForLanguage.id,
databaseCreatedAt: databaseForLanguage.created_at,
commitOid: databaseForLanguage.commit_oid,
};
} catch (e) {
void extLogger.log(`Error: ${getErrorMessage(e)}`);

3
extensions/ql-vscode/src/databases/db-item.ts
View File

@@ -1,5 +1,7 @@
// This file contains models that are used to represent the databases.
import { DatabaseOrigin } from "./local-databases/database-origin";
export enum DbItemKind {
RootLocal = "RootLocal",
LocalList = "LocalList",
@@ -38,6 +40,7 @@ export interface LocalDatabaseDbItem {
databaseName: string;
dateAdded: number;
language: string;
origin: DatabaseOrigin;
storagePath: string;
parentListName?: string;
}

1
extensions/ql-vscode/src/databases/db-tree-creator.ts
View File

@@ -197,6 +197,7 @@ function createLocalDb(
databaseName: db.name,
dateAdded: db.dateAdded,
language: db.language,
origin: db.origin,
storagePath: db.storagePath,
selected: !!selected,
parentListName: listName,

11
extensions/ql-vscode/src/databases/local-databases-ui.ts
View File

@@ -367,6 +367,9 @@ export class DatabaseUI extends DisposableObject {
await this.databaseManager.openDatabase(
uri,
{
type: "folder",
},
makeSelected,
nameOverride,
{
@@ -704,7 +707,9 @@ export class DatabaseUI extends DisposableObject {
this.queryServer?.cliServer,
);
} else {
await this.databaseManager.openDatabase(uri);
await this.databaseManager.openDatabase(uri, {
type: "folder",
});
}
} catch (e) {
// rethrow and let this be handled by default error handling.
@@ -819,7 +824,9 @@ export class DatabaseUI extends DisposableObject {
if (byFolder) {
const fixedUri = await this.fixDbUri(uri);
// we are selecting a database folder
return await this.databaseManager.openDatabase(fixedUri);
return await this.databaseManager.openDatabase(fixedUri, {
type: "folder",
});
} else {
// we are selecting a database archive. Must unzip into a workspace-controlled area
// before importing.

5
extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts
View File

@@ -14,6 +14,7 @@ import { isLikelyDatabaseRoot } from "./db-contents-heuristics";
import { stat } from "fs-extra";
import { containsPath, pathsEqual } from "../../common/files";
import { DatabaseContents } from "./database-contents";
import { DatabaseOrigin } from "./database-origin";
export class DatabaseItemImpl implements DatabaseItem {
// These are only public in the implementation, they are readonly in the interface
@@ -61,6 +62,10 @@ export class DatabaseItemImpl implements DatabaseItem {
return this.options.dateAdded;
}
public get origin(): DatabaseOrigin | undefined {
return this.options.origin;
}
public resolveSourceFile(uriStr: string | undefined): vscode.Uri {
const sourceArchive = this.sourceArchive;
const uri = uriStr ? vscode.Uri.parse(uriStr, true) : undefined;

6
extensions/ql-vscode/src/databases/local-databases/database-item.ts
View File

@@ -2,6 +2,7 @@ import vscode from "vscode";
import * as cli from "../../codeql-cli/cli";
import { DatabaseContents } from "./database-contents";
import { DatabaseOptions } from "./database-options";
import { DatabaseOrigin } from "./database-origin";
/** An item in the list of available databases */
export interface DatabaseItem {
@@ -25,6 +26,11 @@ export interface DatabaseItem {
*/
readonly dateAdded: number | undefined;
/**
* The origin this database item was retrieved from or undefined if unknown.
*/
readonly origin: DatabaseOrigin | undefined;
/** If the database is invalid, describes why. */
readonly error: Error | undefined;

16
extensions/ql-vscode/src/databases/local-databases/database-manager.ts
View File

@@ -35,6 +35,7 @@ import { DatabaseChangedEvent, DatabaseEventKind } from "./database-events";
import { DatabaseResolver } from "./database-resolver";
import { telemetryListener } from "../../common/vscode/telemetry";
import { LanguageContextStore } from "../../language-context-store";
import { DatabaseOrigin } from "./database-origin";
/**
* The name of the key in the workspaceState dictionary in which we
@@ -132,6 +133,7 @@ export class DatabaseManager extends DisposableObject {
*/
public async openDatabase(
uri: vscode.Uri,
origin: DatabaseOrigin | undefined,
makeSelected = true,
displayName?: string,
{
@@ -139,7 +141,11 @@ export class DatabaseManager extends DisposableObject {
addSourceArchiveFolder = addDatabaseSourceToWorkspace(),
}: OpenDatabaseOptions = {},
): Promise<DatabaseItem> {
const databaseItem = await this.createDatabaseItem(uri, displayName);
const databaseItem = await this.createDatabaseItem(
uri,
origin,
displayName,
);
return await this.addExistingDatabaseItem(
databaseItem,
@@ -190,6 +196,7 @@ export class DatabaseManager extends DisposableObject {
*/
private async createDatabaseItem(
uri: vscode.Uri,
origin: DatabaseOrigin | undefined,
displayName: string | undefined,
): Promise<DatabaseItemImpl> {
const contents = await DatabaseResolver.resolveDatabaseContents(uri);
@@ -198,6 +205,7 @@ export class DatabaseManager extends DisposableObject {
displayName,
dateAdded: Date.now(),
language: await this.getPrimaryLanguage(uri.fsPath),
origin,
};
const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions);
@@ -213,6 +221,7 @@ export class DatabaseManager extends DisposableObject {
*/
public async createOrOpenDatabaseItem(
uri: vscode.Uri,
origin: DatabaseOrigin | undefined,
): Promise<DatabaseItem> {
const existingItem = this.findDatabaseItem(uri);
if (existingItem !== undefined) {
@@ -221,7 +230,7 @@ export class DatabaseManager extends DisposableObject {
}
// We don't add this to the list automatically, but the user can add it later.
return this.createDatabaseItem(uri, undefined);
return this.createDatabaseItem(uri, origin, undefined);
}
public async createSkeletonPacks(databaseItem: DatabaseItem) {
@@ -356,6 +365,7 @@ export class DatabaseManager extends DisposableObject {
let displayName: string | undefined = undefined;
let dateAdded = undefined;
let language = undefined;
let origin = undefined;
if (state.options) {
if (typeof state.options.displayName === "string") {
displayName = state.options.displayName;
@@ -364,6 +374,7 @@ export class DatabaseManager extends DisposableObject {
dateAdded = state.options.dateAdded;
}
language = state.options.language;
origin = state.options.origin;
}
const dbBaseUri = vscode.Uri.parse(state.uri, true);
@@ -376,6 +387,7 @@ export class DatabaseManager extends DisposableObject {
displayName,
dateAdded,
language,
origin,
};
const item = new DatabaseItemImpl(dbBaseUri, undefined, fullOptions);

4
extensions/ql-vscode/src/databases/local-databases/database-options.ts
View File

@@ -1,10 +1,14 @@
import { DatabaseOrigin } from "./database-origin";
export interface DatabaseOptions {
displayName?: string;
dateAdded?: number | undefined;
language?: string;
origin?: DatabaseOrigin;
}
export interface FullDatabaseOptions extends DatabaseOptions {
dateAdded: number | undefined;
language: string | undefined;
origin: DatabaseOrigin | undefined;
}

32
extensions/ql-vscode/src/databases/local-databases/database-origin.ts Normal file
View File

@@ -0,0 +1,32 @@
interface DatabaseOriginFolder {
type: "folder";
}
interface DatabaseOriginArchive {
type: "archive";
path: string;
}
interface DatabaseOriginGitHub {
type: "github";
repository: string;
databaseId: number;
databaseCreatedAt: string;
commitOid: string | null;
}
interface DatabaseOriginInternet {
type: "url";
url: string;
}
interface DatabaseOriginDebugger {
type: "debugger";
}
export type DatabaseOrigin =
| DatabaseOriginFolder
| DatabaseOriginArchive
| DatabaseOriginGitHub
| DatabaseOriginInternet
| DatabaseOriginDebugger;

4
extensions/ql-vscode/src/debugger/debugger-ui.ts
View File

@@ -105,7 +105,9 @@ class QLDebugAdapterTracker
body: CodeQLProtocol.EvaluationStartedEvent["body"],
): Promise<void> {
const dbUri = Uri.file(this.configuration.database);
const dbItem = await this.dbm.createOrOpenDatabaseItem(dbUri);
const dbItem = await this.dbm.createOrOpenDatabaseItem(dbUri, {
type: "debugger",
});
// When cancellation is requested from the query history view, we just stop the debug session.
const tokenSource = new CancellationTokenSource();

1
extensions/ql-vscode/src/query-testing/test-runner.ts
View File

@@ -103,6 +103,7 @@ export class TestRunner extends DisposableObject {
try {
const reopenedDatabase = await this.databaseManager.openDatabase(
uri,
closedDatabase.origin,
false,
);
await this.databaseManager.renameDatabaseItem(

3
extensions/ql-vscode/test/factories/databases/databases.ts
View File

@@ -11,6 +11,9 @@ export function mockDbOptions(): FullDatabaseOptions {
return {
dateAdded: 123,
language: "",
origin: {
type: "folder",
},
};
}

6
extensions/ql-vscode/test/factories/db-config-factories.ts
View File

@@ -7,6 +7,7 @@ import {
SelectedDbItem,
DB_CONFIG_VERSION,
} from "../../src/databases/config/db-config";
import { DatabaseOrigin } from "../../src/databases/local-databases/database-origin";
export function createDbConfig({
remoteLists = [],
@@ -45,16 +46,21 @@ export function createLocalDbConfigItem({
dateAdded = faker.date.past().getTime(),
language = `language${faker.number.int()}`,
storagePath = `storagePath${faker.number.int()}`,
origin = {
type: "folder",
},
}: {
name?: string;
dateAdded?: number;
language?: string;
storagePath?: string;
origin?: DatabaseOrigin;
} = {}): LocalDatabase {
return {
name,
dateAdded,
language,
storagePath,
origin,
};
}

6
extensions/ql-vscode/test/factories/db-item-factories.ts
View File

@@ -12,6 +12,7 @@ import {
RootLocalDbItem,
RootRemoteDbItem,
} from "../../src/databases/db-item";
import { DatabaseOrigin } from "../../src/databases/local-databases/database-origin";
// Root Remote Db Items
export function createRootRemoteDbItem({
@@ -124,12 +125,16 @@ export function createLocalDatabaseDbItem({
language = `language${faker.number.int()}`,
storagePath = `storagePath${faker.number.int()}`,
selected = false,
origin = {
type: "folder",
},
}: {
databaseName?: string;
dateAdded?: number;
language?: string;
storagePath?: string;
selected?: boolean;
origin?: DatabaseOrigin;
} = {}): LocalDatabaseDbItem {
return {
kind: DbItemKind.LocalDatabase,
@@ -138,6 +143,7 @@ export function createLocalDatabaseDbItem({
dateAdded,
language,
storagePath,
origin,
};
}

3
extensions/ql-vscode/test/unit-tests/databases/db-item-selection.test.ts
View File

@@ -57,6 +57,9 @@ describe("db item selection", () => {
dateAdded: 1234,
language: "javascript",
storagePath: "/foo/bar",
origin: {
type: "folder",
},
selected: true,
});
});

19
extensions/ql-vscode/test/unit-tests/databases/db-tree-creator.test.ts
View File

@@ -337,12 +337,18 @@ describe("db tree creator", () => {
dateAdded: 1668428293677,
language: QueryLanguage.Cpp,
storagePath: "/path/to/db1/",
origin: {
type: "folder",
},
},
{
name: "db2",
dateAdded: 1668428472731,
language: "cpp",
storagePath: "/path/to/db2/",
origin: {
type: "folder",
},
},
],
},
@@ -354,6 +360,9 @@ describe("db tree creator", () => {
dateAdded: 1668428472731,
language: "ruby",
storagePath: "/path/to/db3/",
origin: {
type: "folder",
},
},
],
},
@@ -380,6 +389,7 @@ describe("db tree creator", () => {
databaseName: db.name,
dateAdded: db.dateAdded,
language: db.language,
origin: db.origin,
storagePath: db.storagePath,
parentListName: dbConfig.databases.local.lists[0].name,
})),
@@ -395,6 +405,7 @@ describe("db tree creator", () => {
databaseName: db.name,
dateAdded: db.dateAdded,
language: db.language,
origin: db.origin,
storagePath: db.storagePath,
parentListName: dbConfig.databases.local.lists[1].name,
})),
@@ -409,12 +420,18 @@ describe("db tree creator", () => {
dateAdded: 1668428293677,
language: "csharp",
storagePath: "/path/to/db1/",
origin: {
type: "folder",
},
},
{
name: "db2",
dateAdded: 1668428472731,
language: "go",
storagePath: "/path/to/db2/",
origin: {
type: "folder",
},
},
],
});
@@ -434,6 +451,7 @@ describe("db tree creator", () => {
databaseName: dbConfig.databases.local.databases[0].name,
dateAdded: dbConfig.databases.local.databases[0].dateAdded,
language: dbConfig.databases.local.databases[0].language,
origin: dbConfig.databases.local.databases[0].origin,
storagePath: dbConfig.databases.local.databases[0].storagePath,
});
expect(localDatabaseNodes[1]).toEqual({
@@ -442,6 +460,7 @@ describe("db tree creator", () => {
databaseName: dbConfig.databases.local.databases[1].name,
dateAdded: dbConfig.databases.local.databases[1].dateAdded,
language: dbConfig.databases.local.databases[1].language,
origin: dbConfig.databases.local.databases[1].origin,
storagePath: dbConfig.databases.local.databases[1].storagePath,
});
});

30
extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts
View File

@@ -186,12 +186,18 @@ describe("db panel rendering nodes", () => {
dateAdded: 1668428293677,
language: QueryLanguage.Cpp,
storagePath: "/path/to/db1/",
origin: {
type: "folder",
},
},
{
name: "db2",
dateAdded: 1668428472731,
language: QueryLanguage.Cpp,
storagePath: "/path/to/db2/",
origin: {
type: "folder",
},
},
],
},
@@ -203,6 +209,9 @@ describe("db panel rendering nodes", () => {
dateAdded: 1668428472731,
language: "ruby",
storagePath: "/path/to/db3/",
origin: {
type: "folder",
},
},
],
},
@@ -238,6 +247,9 @@ describe("db panel rendering nodes", () => {
language: QueryLanguage.Cpp,
storagePath: "/path/to/db1/",
selected: false,
origin: {
type: "folder",
},
},
{
kind: DbItemKind.LocalDatabase,
@@ -246,6 +258,9 @@ describe("db panel rendering nodes", () => {
language: QueryLanguage.Cpp,
storagePath: "/path/to/db2/",
selected: false,
origin: {
type: "folder",
},
},
]);
checkLocalListItem(localListItems[1], "my-list-2", [
@@ -256,6 +271,9 @@ describe("db panel rendering nodes", () => {
language: "ruby",
storagePath: "/path/to/db3/",
selected: false,
origin: {
type: "folder",
},
},
]);
});
@@ -268,12 +286,18 @@ describe("db panel rendering nodes", () => {
dateAdded: 1668428293677,
language: "csharp",
storagePath: "/path/to/db1/",
origin: {
type: "folder",
},
},
{
name: "db2",
dateAdded: 1668428472731,
language: "go",
storagePath: "/path/to/db2/",
origin: {
type: "folder",
},
},
],
});
@@ -306,6 +330,9 @@ describe("db panel rendering nodes", () => {
language: "csharp",
storagePath: "/path/to/db1/",
selected: false,
origin: {
type: "folder",
},
});
checkLocalDatabaseItem(localDatabaseItems[1], {
kind: DbItemKind.LocalDatabase,
@@ -314,6 +341,9 @@ describe("db panel rendering nodes", () => {
language: "go",
storagePath: "/path/to/db2/",
selected: false,
origin: {
type: "folder",
},
});
});
});

34
extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-queries/local-databases.test.ts
View File

@@ -597,6 +597,9 @@ describe("local databases", () => {
const options: FullDatabaseOptions = {
dateAdded: 123,
language,
origin: {
type: "folder",
},
};
mockDbItem = createMockDB(dir, options);
@@ -728,13 +731,19 @@ describe("local databases", () => {
});
it("should resolve the database contents", async () => {
await databaseManager.openDatabase(mockDbItem.databaseUri);
await databaseManager.openDatabase(
mockDbItem.databaseUri,
mockDbItem.origin,
);
expect(resolveDatabaseContentsSpy).toBeCalledTimes(2);
});
it("should set the database as the currently selected one", async () => {
await databaseManager.openDatabase(mockDbItem.databaseUri);
await databaseManager.openDatabase(
mockDbItem.databaseUri,
mockDbItem.origin,
);
expect(setCurrentDatabaseItemSpy).toBeCalledTimes(1);
});
@@ -742,7 +751,10 @@ describe("local databases", () => {
it("should not add database source archive folder when `codeQL.addingDatabases.addDatabaseSourceToWorkspace` is `false`", async () => {
jest.spyOn(config, "addDatabaseSourceToWorkspace").mockReturnValue(false);
await databaseManager.openDatabase(mockDbItem.databaseUri);
await databaseManager.openDatabase(
mockDbItem.databaseUri,
mockDbItem.origin,
);
expect(addDatabaseSourceArchiveFolderSpy).toBeCalledTimes(0);
});
@@ -750,7 +762,10 @@ describe("local databases", () => {
it("should add database source archive folder when `codeQL.addingDatabases.addDatabaseSourceToWorkspace` is `true`", async () => {
jest.spyOn(config, "addDatabaseSourceToWorkspace").mockReturnValue(true);
await databaseManager.openDatabase(mockDbItem.databaseUri);
await databaseManager.openDatabase(
mockDbItem.databaseUri,
mockDbItem.origin,
);
expect(addDatabaseSourceArchiveFolderSpy).toBeCalledTimes(1);
});
@@ -766,6 +781,7 @@ describe("local databases", () => {
await databaseManager.openDatabase(
mockDbItem.databaseUri,
mockDbItem.origin,
makeSelected,
nameOverride,
{ isTutorialDatabase },
@@ -779,7 +795,10 @@ describe("local databases", () => {
it("should create a skeleton QL pack", async () => {
jest.spyOn(config, "isCodespacesTemplate").mockReturnValue(true);
await databaseManager.openDatabase(mockDbItem.databaseUri);
await databaseManager.openDatabase(
mockDbItem.databaseUri,
mockDbItem.origin,
);
expect(createSkeletonPacksSpy).toBeCalledTimes(1);
});
@@ -790,7 +809,10 @@ describe("local databases", () => {
it("should not create a skeleton QL pack", async () => {
jest.spyOn(config, "isCodespacesTemplate").mockReturnValue(false);
await databaseManager.openDatabase(mockDbItem.databaseUri);
await databaseManager.openDatabase(
mockDbItem.databaseUri,
mockDbItem.origin,
);
expect(createSkeletonPacksSpy).toBeCalledTimes(0);
});
});

11
extensions/ql-vscode/test/vscode-tests/no-workspace/query-testing/test-runner.test.ts
View File

@@ -39,12 +39,18 @@ describe("test-runner", () => {
const preTestDatabaseItem = new DatabaseItemImpl(
Uri.file("/path/to/test/dir/dir.testproj"),
undefined,
mockedObject<FullDatabaseOptions>({ displayName: "custom display name" }),
mockedObject<FullDatabaseOptions>({
displayName: "custom display name",
origin: { type: "folder" },
}),
);
const postTestDatabaseItem = new DatabaseItemImpl(
Uri.file("/path/to/test/dir/dir.testproj"),
undefined,
mockedObject<FullDatabaseOptions>({ displayName: "default name" }),
mockedObject<FullDatabaseOptions>({
displayName: "default name",
origin: { type: "folder" },
}),
);
beforeEach(() => {
@@ -160,6 +166,7 @@ describe("test-runner", () => {
expect(openDatabaseSpy).toBeCalledTimes(1);
expect(openDatabaseSpy).toBeCalledWith(
preTestDatabaseItem.databaseUri,
preTestDatabaseItem.origin,
false,
);