Merge pull request #2055 from github/elena/download-ql-packs

Generate QL pack for CodeTour

extensions/ql-vscode/src/cli.ts

@@ -28,6 +28,7 @@ import { CompilationMessage } from "./pure/legacy-messages";
 import { sarifParser } from "./sarif-parser";
 import { dbSchemeToLanguage, walkDirectory } from "./helpers";
 import { App } from "./common/app";
+import { QueryLanguage } from "./qlpack-generator";
 
 /**
  * The version of the SARIF format that we are using.
@@ -1216,6 +1217,23 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  /**
+   * Adds a core language QL library pack for the given query language as a dependency
+   * of the current package, and then installs them. This command modifies the qlpack.yml
+   * file of the current package. Formatting and comments will be removed.
+   * @param dir The directory where QL pack exists.
+   * @param language The language of the QL pack.
+   */
+  async packAdd(dir: string, queryLanguage: QueryLanguage) {
+    const args = ["--dir", dir];
+    args.push(`codeql/${queryLanguage}-all`);
+    return this.runJsonCodeQlCliCommandWithAuthentication(
+      ["pack", "add"],
+      args,
+      `Adding and installing ${queryLanguage} pack dependency.`,
+    );
+  }
+
   /**
    * Downloads a specified pack.
    * @param packs The `<package-scope/name[@version]>` of the packs to download.

extensions/ql-vscode/src/databases.ts

@@ -26,6 +26,7 @@ import { QueryRunner } from "./queryRunner";
 import { pathsEqual } from "./pure/files";
 import { redactableError } from "./pure/errors";
 import { isCodespacesTemplate } from "./config";
+import { QlPackGenerator, QueryLanguage } from "./qlpack-generator";
 
 /**
  * databases.ts
@@ -655,9 +656,27 @@ export class DatabaseManager extends DisposableObject {
       return;
     }
 
-    await showBinaryChoiceDialog(
-      `We've noticed you don't have a QL pack downloaded to analyze this database. Can we set up a ${databaseItem.language} query pack for you`,
+    const answer = await showBinaryChoiceDialog(
+      `We've noticed you don't have a CodeQL pack available to analyze this database. Can we set up a query pack for you?`,
     );
+
+    if (!answer) {
+      return;
+    }
+
+    try {
+      const qlPackGenerator = new QlPackGenerator(
+        folderName,
+        databaseItem.language as QueryLanguage,
+        this.cli,
+        this.ctx.storageUri?.fsPath,
+      );
+      await qlPackGenerator.generate();
+    } catch (e: unknown) {
+      void this.logger.log(
+        `Could not create skeleton QL pack: ${getErrorMessage(e)}`,
+      );
+    }
   }
 
   private async reregisterDatabases(

extensions/ql-vscode/src/qlpack-generator.ts

@@ -0,0 +1,101 @@
+import { writeFile } from "fs-extra";
+import { dump } from "js-yaml";
+import { join } from "path";
+import { Uri, workspace } from "vscode";
+import { CodeQLCliServer } from "./cli";
+
+export type QueryLanguage =
+  | "csharp"
+  | "cpp"
+  | "go"
+  | "java"
+  | "javascript"
+  | "python"
+  | "ruby"
+  | "swift";
+
+export class QlPackGenerator {
+  private readonly qlpackName: string;
+  private readonly qlpackVersion: string;
+  private readonly header: string;
+  private readonly qlpackFileName: string;
+  private readonly folderUri: Uri;
+
+  constructor(
+    private readonly folderName: string,
+    private readonly queryLanguage: QueryLanguage,
+    private readonly cliServer: CodeQLCliServer,
+    private readonly storagePath: string | undefined,
+  ) {
+    if (this.storagePath === undefined) {
+      throw new Error("Workspace storage path is undefined");
+    }
+    this.qlpackName = `getting-started/codeql-extra-queries-${this.queryLanguage}`;
+    this.qlpackVersion = "1.0.0";
+    this.header = "# This is an automatically generated file.\n\n";
+
+    this.qlpackFileName = "qlpack.yml";
+    this.folderUri = Uri.file(join(this.storagePath, this.folderName));
+  }
+
+  public async generate() {
+    // create QL pack folder and add to workspace
+    await this.createWorkspaceFolder();
+
+    // create qlpack.yml
+    await this.createQlPackYaml();
+
+    // create example.ql
+    await this.createExampleQlFile();
+
+    // create codeql-pack.lock.yml
+    await this.createCodeqlPackLockYaml();
+  }
+
+  private async createWorkspaceFolder() {
+    await workspace.fs.createDirectory(this.folderUri);
+
+    const end = (workspace.workspaceFolders || []).length;
+
+    await workspace.updateWorkspaceFolders(end, 0, {
+      name: this.folderName,
+      uri: this.folderUri,
+    });
+  }
+
+  private async createQlPackYaml() {
+    const qlPackFilePath = join(this.folderUri.fsPath, this.qlpackFileName);
+
+    const qlPackYml = {
+      name: this.qlpackName,
+      version: this.qlpackVersion,
+      dependencies: {},
+    };
+
+    await writeFile(qlPackFilePath, this.header + dump(qlPackYml), "utf8");
+  }
+
+  private async createExampleQlFile() {
+    const exampleQlFilePath = join(this.folderUri.fsPath, "example.ql");
+
+    const exampleQl = `
+/**
+ * This is an automatically generated file
+ * @name Empty block
+ * @kind problem
+ * @problem.severity warning
+ * @id ${this.queryLanguage}/example/empty-block
+ */
+
+import ${this.queryLanguage}
+
+select "Hello, world!"
+`.trim();
+
+    await writeFile(exampleQlFilePath, exampleQl, "utf8");
+  }
+
+  private async createCodeqlPackLockYaml() {
+    await this.cliServer.packAdd(this.folderUri.fsPath, this.queryLanguage);
+  }
+}

extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases.test.ts

@@ -23,6 +23,7 @@ import { testDisposeHandler } from "../test-dispose-handler";
 import { QueryRunner } from "../../../src/queryRunner";
 import * as helpers from "../../../src/helpers";
 import { Setting } from "../../../src/config";
+import { QlPackGenerator } from "../../../src/qlpack-generator";
 
 describe("databases", () => {
   const MOCK_DB_OPTIONS: FullDatabaseOptions = {
@@ -32,11 +33,13 @@ describe("databases", () => {
   };
 
   let databaseManager: DatabaseManager;
+  let extensionContext: ExtensionContext;
 
   let updateSpy: jest.Mock<Promise<void>, []>;
   let registerSpy: jest.Mock<Promise<void>, []>;
   let deregisterSpy: jest.Mock<Promise<void>, []>;
   let resolveDatabaseSpy: jest.Mock<Promise<DbInfo>, []>;
+  let packAddSpy: jest.Mock<any, []>;
   let logSpy: jest.Mock<any, []>;
 
   let showBinaryChoiceDialogSpy: jest.SpiedFunction<
@@ -52,6 +55,7 @@ describe("databases", () => {
     registerSpy = jest.fn(() => Promise.resolve(undefined));
     deregisterSpy = jest.fn(() => Promise.resolve(undefined));
     resolveDatabaseSpy = jest.fn(() => Promise.resolve({} as DbInfo));
+    packAddSpy = jest.fn();
     logSpy = jest.fn(() => {
       /* */
     });
@@ -60,16 +64,19 @@ describe("databases", () => {
       .spyOn(helpers, "showBinaryChoiceDialog")
       .mockResolvedValue(true);
 
+    extensionContext = {
+      workspaceState: {
+        update: updateSpy,
+        get: () => [],
+      },
+      // pretend like databases added in the temp dir are controlled by the extension
+      // so that they are deleted upon removal
+      storagePath: dir.name,
+      storageUri: Uri.parse(dir.name),
+    } as unknown as ExtensionContext;
+
     databaseManager = new DatabaseManager(
-      {
-        workspaceState: {
-          update: updateSpy,
-          get: () => [],
-        },
-        // pretend like databases added in the temp dir are controlled by the extension
-        // so that they are deleted upon removal
-        storagePath: dir.name,
-      } as unknown as ExtensionContext,
+      extensionContext,
       {
         registerDatabase: registerSpy,
         deregisterDatabase: deregisterSpy,
@@ -79,6 +86,7 @@ describe("databases", () => {
       } as unknown as QueryRunner,
       {
         resolveDatabase: resolveDatabaseSpy,
+        packAdd: packAddSpy,
       } as unknown as CodeQLCliServer,
       {
         log: logSpy,
@@ -589,20 +597,46 @@ describe("databases", () => {
 
   describe("createSkeletonPacks", () => {
     let mockDbItem: DatabaseItemImpl;
+    let language: string;
+    let generateSpy: jest.SpyInstance;
+
+    beforeEach(() => {
+      language = "ruby";
+
+      const options: FullDatabaseOptions = {
+        dateAdded: 123,
+        ignoreSourceArchive: false,
+        language,
+      };
+      mockDbItem = createMockDB(options);
+
+      generateSpy = jest
+        .spyOn(QlPackGenerator.prototype, "generate")
+        .mockImplementation(() => Promise.resolve());
+    });
 
     describe("when the language is set", () => {
       it("should offer the user to set up a skeleton QL pack", async () => {
-        const options: FullDatabaseOptions = {
-          dateAdded: 123,
-          ignoreSourceArchive: false,
-          language: "ruby",
-        };
-        mockDbItem = createMockDB(options);
-
         await (databaseManager as any).createSkeletonPacks(mockDbItem);
 
         expect(showBinaryChoiceDialogSpy).toBeCalledTimes(1);
       });
+
+      it("should return early if the user refuses help", async () => {
+        showBinaryChoiceDialogSpy = jest
+          .spyOn(helpers, "showBinaryChoiceDialog")
+          .mockResolvedValue(false);
+
+        await (databaseManager as any).createSkeletonPacks(mockDbItem);
+
+        expect(generateSpy).not.toBeCalled();
+      });
+
+      it("should create the skeleton QL pack for the user", async () => {
+        await (databaseManager as any).createSkeletonPacks(mockDbItem);
+
+        expect(generateSpy).toBeCalled();
+      });
     });
 
     describe("when the language is not set", () => {

extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts

@@ -0,0 +1,74 @@
+import { join } from "path";
+import { existsSync } from "fs";
+import { QlPackGenerator, QueryLanguage } from "../../../src/qlpack-generator";
+import { CodeQLCliServer } from "../../../src/cli";
+import { Uri, workspace } from "vscode";
+import { getErrorMessage } from "../../../src/pure/helpers-pure";
+import * as tmp from "tmp";
+
+describe("QlPackGenerator", () => {
+  let packFolderName: string;
+  let packFolderPath: string;
+  let qlPackYamlFilePath: string;
+  let exampleQlFilePath: string;
+  let language: string;
+  let generator: QlPackGenerator;
+  let packAddSpy: jest.SpyInstance;
+  let dir: tmp.DirResult;
+
+  beforeEach(async () => {
+    dir = tmp.dirSync();
+
+    language = "ruby";
+    packFolderName = `test-ql-pack-${language}`;
+    packFolderPath = Uri.file(join(dir.name, packFolderName)).fsPath;
+
+    qlPackYamlFilePath = join(packFolderPath, "qlpack.yml");
+    exampleQlFilePath = join(packFolderPath, "example.ql");
+
+    packAddSpy = jest.fn();
+    const mockCli = {
+      packAdd: packAddSpy,
+    } as unknown as CodeQLCliServer;
+
+    generator = new QlPackGenerator(
+      packFolderName,
+      language as QueryLanguage,
+      mockCli,
+      dir.name,
+    );
+  });
+
+  afterEach(async () => {
+    try {
+      dir.removeCallback();
+
+      const workspaceFolders = workspace.workspaceFolders || [];
+      const folderIndex = workspaceFolders.findIndex(
+        (workspaceFolder) => workspaceFolder.name === dir.name,
+      );
+
+      if (folderIndex !== undefined) {
+        workspace.updateWorkspaceFolders(folderIndex, 1);
+      }
+    } catch (e) {
+      console.log(
+        `Could not remove folder from workspace: ${getErrorMessage(e)}`,
+      );
+    }
+  });
+
+  it("should generate a QL pack", async () => {
+    expect(existsSync(packFolderPath)).toBe(false);
+    expect(existsSync(qlPackYamlFilePath)).toBe(false);
+    expect(existsSync(exampleQlFilePath)).toBe(false);
+
+    await generator.generate();
+
+    expect(existsSync(packFolderPath)).toBe(true);
+    expect(existsSync(qlPackYamlFilePath)).toBe(true);
+    expect(existsSync(exampleQlFilePath)).toBe(true);
+
+    expect(packAddSpy).toHaveBeenCalledWith(packFolderPath, language);
+  });
+});