Merge pull request #3037 from github/koesie10/detect-language

Detect existing query packs when creating skeleton query

extensions/ql-vscode/src/codeql-cli/cli.ts

@@ -1244,11 +1244,13 @@ export class CodeQLCliServer implements Disposable {
    * @param additionalPacks A list of directories to search for qlpacks.
    * @param extensionPacksOnly Whether to only search for extension packs. If true, only extension packs will
    *    be returned. If false, all packs will be returned.
+   * @param kind Whether to only search for qlpacks with a certain kind.
    * @returns A dictionary mapping qlpack name to the directory it comes from
    */
   async resolveQlpacks(
     additionalPacks: string[],
     extensionPacksOnly = false,
+    kind?: "query" | "library" | "all",
   ): Promise<QlpacksInfo> {
     const args = this.getAdditionalPacksArg(additionalPacks);
     if (extensionPacksOnly) {
@@ -1259,6 +1261,8 @@ export class CodeQLCliServer implements Disposable {
         return {};
       }
       args.push("--kind", "extension", "--no-recursive");
+    } else if (kind) {
+      args.push("--kind", kind);
     }
 
     return this.runJsonCodeQlCliCommand<QlpacksInfo>(

extensions/ql-vscode/src/databases/local-databases/database-manager.ts

@@ -274,10 +274,9 @@ export class DatabaseManager extends DisposableObject {
 
     try {
       const qlPackGenerator = new QlPackGenerator(
-        folderName,
         databaseItem.language,
         this.cli,
-        firstWorkspaceFolder,
+        join(firstWorkspaceFolder, folderName),
       );
       await qlPackGenerator.generate();
     } catch (e: unknown) {

extensions/ql-vscode/src/local-queries/qlpack-generator.ts

@@ -13,20 +13,16 @@ export class QlPackGenerator {
   private readonly folderUri: Uri;
 
   constructor(
-    private readonly folderName: string,
     private readonly queryLanguage: QueryLanguage,
     private readonly cliServer: CodeQLCliServer,
-    private readonly storagePath: string | undefined,
+    private readonly storagePath: string,
   ) {
-    if (this.storagePath === undefined) {
-      throw new Error("Workspace storage path is undefined");
-    }
     this.qlpackName = `getting-started/codeql-extra-queries-${this.queryLanguage}`;
     this.qlpackVersion = "1.0.0";
     this.header = "# This is an automatically generated file.\n\n";
 
     this.qlpackFileName = "codeql-pack.yml";
-    this.folderUri = Uri.file(join(this.storagePath, this.folderName));
+    this.folderUri = Uri.file(this.storagePath);
   }
 
   public async generate() {

extensions/ql-vscode/src/local-queries/skeleton-query-wizard.ts

@@ -1,4 +1,4 @@
-import { basename, dirname, join } from "path";
+import { dirname, join } from "path";
 import { Uri, window, window as Window, workspace } from "vscode";
 import { CodeQLCliServer } from "../codeql-cli/cli";
 import { showAndLogExceptionWithTelemetry } from "../common/logging";
@@ -7,7 +7,10 @@ import {
   getLanguageDisplayName,
   QueryLanguage,
 } from "../common/query-language";
-import { getFirstWorkspaceFolder } from "../common/vscode/workspace-folders";
+import {
+  getFirstWorkspaceFolder,
+  getOnDiskWorkspaceFolders,
+} from "../common/vscode/workspace-folders";
 import { asError, getErrorMessage } from "../common/helpers-pure";
 import { QlPackGenerator } from "./qlpack-generator";
 import { DatabaseItem, DatabaseManager } from "../databases/local-databases";
@@ -25,12 +28,16 @@ import {
   isCodespacesTemplate,
   setQlPackLocation,
 } from "../config";
-import { lstat, pathExists } from "fs-extra";
+import { lstat, pathExists, readFile } from "fs-extra";
 import { askForLanguage } from "../codeql-cli/query-language";
 import { showInformationMessageWithAction } from "../common/vscode/dialog";
 import { redactableError } from "../common/errors";
 import { App } from "../common/app";
 import { QueryTreeViewItem } from "../queries-panel/query-tree-view-item";
+import { containsPath } from "../common/files";
+import { getQlPackPath } from "../common/ql";
+import { load } from "js-yaml";
+import { QlPackFile } from "../packaging/qlpack-file";
 
 type QueryLanguagesToDatabaseMap = Record<string, string>;
 
@@ -48,6 +55,7 @@ export const QUERY_LANGUAGE_TO_DATABASE_REPO: QueryLanguagesToDatabaseMap = {
 export class SkeletonQueryWizard {
   private fileName = "example.ql";
   private qlPackStoragePath: string | undefined;
+  private queryStoragePath: string | undefined;
   private downloadPromise: Promise<void> | undefined;
 
   constructor(
@@ -61,10 +69,6 @@ export class SkeletonQueryWizard {
     private language: QueryLanguage | undefined = undefined,
   ) {}
 
-  private get folderName() {
-    return `codeql-custom-queries-${this.language}`;
-  }
-
   /**
    * Wait for the download process to complete by waiting for the user to select
    * either "Download database" or closing the dialog. This is used for testing.
@@ -76,6 +80,14 @@ export class SkeletonQueryWizard {
   }
 
   public async execute() {
+    // First try detecting the language based on the existing qlpacks.
+    // This will override the selected language if there is an existing query pack.
+    const detectedLanguage = await this.detectLanguage();
+    if (detectedLanguage) {
+      this.language = detectedLanguage;
+    }
+
+    // If no existing qlpack was found, we need to ask the user for the language
     if (!this.language) {
       // show quick pick to choose language
       this.language = await this.chooseLanguage();
@@ -85,18 +97,39 @@ export class SkeletonQueryWizard {
       return;
     }
 
-    this.qlPackStoragePath = await this.determineStoragePath();
+    let createSkeletonQueryPack: boolean = false;
 
-    const skeletonPackAlreadyExists = await pathExists(
+    if (!this.qlPackStoragePath) {
-      join(this.qlPackStoragePath, this.folderName),
+      // This means no existing qlpack was detected in the selected folder, so we need
-    );
+      // to find a new location to store the qlpack. This new location could potentially
+      // already exist.
+      const storagePath = await this.determineStoragePath();
+      this.qlPackStoragePath = join(
+        storagePath,
+        `codeql-custom-queries-${this.language}`,
+      );
 
-    if (skeletonPackAlreadyExists) {
+      // Try to detect if there is already a qlpack in this location. We will assume that
-      // just create a new example query file in skeleton QL pack
+      // the user hasn't changed the language of the qlpack.
-      await this.createExampleFile();
+      const qlPackPath = await getQlPackPath(this.qlPackStoragePath);
+
+      // If we are creating or using a qlpack in the user's selected folder, we will also
+      // create the query in that folder
+      this.queryStoragePath = this.qlPackStoragePath;
+
+      createSkeletonQueryPack = qlPackPath === undefined;
     } else {
+      // A query pack was detected in the selected folder or one of its ancestors, so we
+      // directly use the selected folder as the storage path for the query.
+      this.queryStoragePath = await this.determineStoragePathFromSelection();
+    }
+
+    if (createSkeletonQueryPack) {
       // generate a new skeleton QL pack with query file
       await this.createQlPack();
+    } else {
+      // just create a new example query file in skeleton QL pack
+      await this.createExampleFile();
     }
 
     // open the query file
@@ -113,13 +146,11 @@ export class SkeletonQueryWizard {
   }
 
   private async openExampleFile() {
-    if (this.folderName === undefined || this.qlPackStoragePath === undefined) {
+    if (this.queryStoragePath === undefined) {
       throw new Error("Path to folder is undefined");
     }
 
-    const queryFileUri = Uri.file(
+    const queryFileUri = Uri.file(join(this.queryStoragePath, this.fileName));
-      join(this.qlPackStoragePath, this.folderName, this.fileName),
-    );
 
     void workspace.openTextDocument(queryFileUri).then((doc) => {
       void Window.showTextDocument(doc, {
@@ -133,15 +164,7 @@ export class SkeletonQueryWizard {
       return this.determineRootStoragePath();
     }
 
-    const storagePath = await this.determineStoragePathFromSelection();
+    return this.determineStoragePathFromSelection();
-
-    // If the user has selected a folder or file within a folder that matches the current
-    // folder name, we should create a query rather than a query pack
-    if (basename(storagePath) === this.folderName) {
-      return dirname(storagePath);
-    }
-
-    return storagePath;
   }
 
   private async determineStoragePathFromSelection(): Promise<string> {
@@ -194,6 +217,62 @@ export class SkeletonQueryWizard {
     return storageFolder;
   }
 
+  private async detectLanguage(): Promise<QueryLanguage | undefined> {
+    if (this.selectedItems.length < 1) {
+      return undefined;
+    }
+
+    this.progress({
+      message: "Resolving existing query packs",
+      step: 1,
+      maxStep: 3,
+    });
+
+    const storagePath = await this.determineStoragePathFromSelection();
+
+    const queryPacks = await this.cliServer.resolveQlpacks(
+      getOnDiskWorkspaceFolders(),
+      false,
+      "query",
+    );
+
+    const matchingQueryPacks = Object.values(queryPacks)
+      .map((paths) => paths.find((path) => containsPath(path, storagePath)))
+      .filter((path): path is string => path !== undefined)
+      // Find the longest matching path
+      .sort((a, b) => b.length - a.length);
+
+    if (matchingQueryPacks.length === 0) {
+      return undefined;
+    }
+
+    const matchingQueryPackPath = matchingQueryPacks[0];
+
+    const qlPackPath = await getQlPackPath(matchingQueryPackPath);
+    if (!qlPackPath) {
+      return undefined;
+    }
+
+    const qlPack = load(await readFile(qlPackPath, "utf8")) as
+      | QlPackFile
+      | undefined;
+    const dependencies = qlPack?.dependencies;
+    if (!dependencies || typeof dependencies !== "object") {
+      return;
+    }
+
+    const matchingLanguages = Object.values(QueryLanguage).filter(
+      (language) => `codeql/${language}-all` in dependencies,
+    );
+    if (matchingLanguages.length !== 1) {
+      return undefined;
+    }
+
+    this.qlPackStoragePath = matchingQueryPackPath;
+
+    return matchingLanguages[0];
+  }
+
   private async chooseLanguage() {
     this.progress({
       message: "Choose language",
@@ -205,8 +284,8 @@ export class SkeletonQueryWizard {
   }
 
   private async createQlPack() {
-    if (this.folderName === undefined) {
+    if (this.qlPackStoragePath === undefined) {
-      throw new Error("Folder name is undefined");
+      throw new Error("Query pack storage path is undefined");
     }
     if (this.language === undefined) {
       throw new Error("Language is undefined");
@@ -220,7 +299,6 @@ export class SkeletonQueryWizard {
 
     try {
       const qlPackGenerator = new QlPackGenerator(
-        this.folderName,
         this.language,
         this.cliServer,
         this.qlPackStoragePath,
@@ -235,7 +313,7 @@ export class SkeletonQueryWizard {
   }
 
   private async createExampleFile() {
-    if (this.folderName === undefined) {
+    if (this.qlPackStoragePath === undefined) {
       throw new Error("Folder name is undefined");
     }
     if (this.language === undefined) {
@@ -251,13 +329,12 @@ export class SkeletonQueryWizard {
 
     try {
       const qlPackGenerator = new QlPackGenerator(
-        this.folderName,
         this.language,
         this.cliServer,
         this.qlPackStoragePath,
       );
 
-      this.fileName = await this.determineNextFileName(this.folderName);
+      this.fileName = await this.determineNextFileName();
       await qlPackGenerator.createExampleQlFile(this.fileName);
     } catch (e: unknown) {
       void this.app.logger.log(
@@ -266,13 +343,18 @@ export class SkeletonQueryWizard {
     }
   }
 
-  private async determineNextFileName(folderName: string): Promise<string> {
+  private async determineNextFileName(): Promise<string> {
-    if (this.qlPackStoragePath === undefined) {
+    if (this.queryStoragePath === undefined) {
-      throw new Error("QL Pack storage path is undefined");
+      throw new Error("Query storage path is undefined");
     }
 
-    const folderUri = Uri.file(join(this.qlPackStoragePath, folderName));
+    const folderUri = Uri.file(this.queryStoragePath);
     const files = await workspace.fs.readDirectory(folderUri);
+    // If the example.ql file doesn't exist yet, use that name
+    if (!files.some(([filename, _fileType]) => filename === this.fileName)) {
+      return this.fileName;
+    }
+
     const qlFiles = files.filter(([filename, _fileType]) =>
       filename.match(/^example[0-9]*\.ql$/),
     );
@@ -281,10 +363,6 @@ export class SkeletonQueryWizard {
   }
 
   private async promptDownloadDatabase() {
-    if (this.qlPackStoragePath === undefined) {
-      throw new Error("QL Pack storage path is undefined");
-    }
-
     if (this.language === undefined) {
       throw new Error("Language is undefined");
     }
@@ -321,10 +399,6 @@ export class SkeletonQueryWizard {
   }
 
   private async downloadDatabase(progress: ProgressCallback) {
-    if (this.qlPackStoragePath === undefined) {
-      throw new Error("QL Pack storage path is undefined");
-    }
-
     if (this.databaseStoragePath === undefined) {
       throw new Error("Database storage path is undefined");
     }
@@ -362,10 +436,6 @@ export class SkeletonQueryWizard {
       throw new Error("Language is undefined");
     }
 
-    if (this.qlPackStoragePath === undefined) {
-      throw new Error("QL Pack storage path is undefined");
-    }
-
     const existingDatabaseItem =
       await SkeletonQueryWizard.findExistingDatabaseItem(
         this.language,
@@ -393,15 +463,11 @@ export class SkeletonQueryWizard {
   }
 
   private get openFileMarkdownLink() {
-    if (this.qlPackStoragePath === undefined) {
+    if (this.queryStoragePath === undefined) {
       throw new Error("QL Pack storage path is undefined");
     }
 
-    const queryPath = join(
+    const queryPath = join(this.queryStoragePath, this.fileName);
-      this.qlPackStoragePath,
-      this.folderName,
-      this.fileName,
-    );
     const queryPathUri = Uri.file(queryPath);
 
     const openFileArgs = [queryPathUri.toString(true)];

extensions/ql-vscode/test/vscode-tests/cli-integration/local-queries/skeleton-query-wizard.test.ts

@@ -8,6 +8,7 @@ import * as tmp from "tmp";
 import {
   MessageItem,
   TextDocument,
+  Uri,
   window,
   workspace,
   WorkspaceFolder,
@@ -18,6 +19,7 @@ import {
   ensureDir,
   ensureDirSync,
   ensureFile,
+  outputFile,
   removeSync,
 } from "fs-extra";
 import { dirname, join } from "path";
@@ -39,6 +41,7 @@ import {
   createQueryTreeFolderItem,
   QueryTreeViewItem,
 } from "../../../../src/queries-panel/query-tree-view-item";
+import { dump } from "js-yaml";
 
 describe("SkeletonQueryWizard", () => {
   let mockCli: CodeQLCliServer;
@@ -67,12 +70,19 @@ describe("SkeletonQueryWizard", () => {
   let openTextDocumentSpy: jest.SpiedFunction<
     typeof workspace.openTextDocument
   >;
+  let resolveQlpacksMock: jest.MockedFunction<
+    typeof CodeQLCliServer.prototype.resolveQlpacks
+  >;
 
   const credentials = testCredentialsWithStub();
   const chosenLanguage = "ruby";
   const selectedItems: QueryTreeViewItem[] = [];
 
   beforeEach(async () => {
+    resolveQlpacksMock = jest.fn().mockImplementation(() => {
+      throw new Error("Not implemented");
+    });
+
     mockCli = mockedObject<CodeQLCliServer>({
       getSupportedLanguages: jest
         .fn()
@@ -85,6 +95,7 @@ describe("SkeletonQueryWizard", () => {
           "csharp",
           "cpp",
         ]),
+      resolveQlpacks: resolveQlpacksMock,
     });
     mockApp = createMockApp();
 
@@ -232,12 +243,16 @@ describe("SkeletonQueryWizard", () => {
   });
 
   describe("if QL pack exists", () => {
+    let qlPackPath: string;
+
     beforeEach(async () => {
       // create a skeleton codeql-custom-queries-${language} folder
       // with an example QL file inside
-      ensureDirSync(
+
-        join(dir.name, `codeql-custom-queries-${chosenLanguage}`, "example.ql"),
+      qlPackPath = join(dir.name, `codeql-custom-queries-${chosenLanguage}`);
-      );
+
+      await ensureFile(join(qlPackPath, "qlpack.yml"));
+      await ensureFile(join(qlPackPath, "example.ql"));
     });
 
     it("should create new query file in the same QL pack folder", async () => {
@@ -267,7 +282,7 @@ describe("SkeletonQueryWizard", () => {
         );
         await wizard.execute();
 
-        expect(createExampleQlFileSpy).toHaveBeenCalledWith("example1.ql");
+        expect(createExampleQlFileSpy).toHaveBeenCalledWith("example.ql");
       });
 
       it("should open the query file", async () => {
@@ -282,9 +297,7 @@ describe("SkeletonQueryWizard", () => {
         await wizard.execute();
 
         expect(openTextDocumentSpy).toHaveBeenCalledWith(
-          expect.objectContaining({
+          Uri.joinPath(Uri.file(qlPackPath), "example.ql"),
-            path: expect.stringMatching("example1.ql"),
-          }),
         );
       });
     });
@@ -459,6 +472,145 @@ describe("SkeletonQueryWizard", () => {
     });
   });
 
+  describe("if selected QL pack exists with different language", () => {
+    let qlPackPath: string;
+    let selectedItems: QueryTreeViewItem[];
+
+    beforeEach(async () => {
+      // create a skeleton codeql-custom-queries-${language} folder
+      // with an example QL file inside
+
+      qlPackPath = join(dir.name, "my-custom-queries-swift");
+
+      await outputFile(
+        join(qlPackPath, "qlpack.yml"),
+        dump({
+          name: "getting-started/my-custom-queries-swift",
+          version: "1.0.0",
+          dependencies: {
+            "codeql/swift-all": "*",
+          },
+        }),
+        "utf-8",
+      );
+      await ensureFile(join(qlPackPath, "example.ql"));
+
+      resolveQlpacksMock.mockResolvedValue({
+        "my/root-pack": [dir.name],
+        "getting-started/my-custom-queries-swift": [qlPackPath],
+      });
+
+      selectedItems = [
+        createQueryTreeFolderItem("folder", qlPackPath, [
+          createQueryTreeFileItem(
+            "example.ql",
+            join(qlPackPath, "example.ql"),
+            "swift",
+          ),
+        ]),
+      ];
+
+      wizard = new SkeletonQueryWizard(
+        mockCli,
+        jest.fn(),
+        credentials,
+        mockApp,
+        mockDatabaseManager,
+        storagePath,
+        selectedItems,
+        QueryLanguage.Javascript,
+      );
+    });
+
+    it("should create new query file in the same QL pack folder", async () => {
+      await wizard.execute();
+
+      expect(createExampleQlFileSpy).toHaveBeenCalledWith("example2.ql");
+    });
+
+    it("should only take into account example QL files", async () => {
+      createFileSync(
+        join(dir.name, `codeql-custom-queries-${chosenLanguage}`, "MyQuery.ql"),
+      );
+
+      await wizard.execute();
+
+      expect(createExampleQlFileSpy).toHaveBeenCalledWith("example2.ql");
+    });
+
+    describe("when qlpack has no language dependencies", () => {
+      beforeEach(async () => {
+        await outputFile(
+          join(qlPackPath, "qlpack.yml"),
+          dump({
+            name: "getting-started/my-custom-queries-swift",
+            version: "1.0.0",
+          }),
+          "utf-8",
+        );
+      });
+
+      it("should open query file in a new QL pack folder", async () => {
+        await wizard.execute();
+
+        expect(openTextDocumentSpy).toHaveBeenCalledWith(
+          Uri.joinPath(
+            Uri.file(qlPackPath),
+            "codeql-custom-queries-javascript",
+            "example.ql",
+          ),
+        );
+      });
+    });
+
+    describe("when qlpack has multiple language dependencies", () => {
+      beforeEach(async () => {
+        await outputFile(
+          join(qlPackPath, "qlpack.yml"),
+          dump({
+            name: "getting-started/my-custom-queries-swift",
+            version: "1.0.0",
+            dependencies: {
+              "codeql/java-all": "*",
+              "codeql/swift-all": "*",
+            },
+          }),
+          "utf-8",
+        );
+      });
+
+      it("should open query file in a new QL pack folder", async () => {
+        await wizard.execute();
+
+        expect(openTextDocumentSpy).toHaveBeenCalledWith(
+          Uri.joinPath(
+            Uri.file(qlPackPath),
+            "codeql-custom-queries-javascript",
+            "example.ql",
+          ),
+        );
+      });
+    });
+
+    describe("when qlpack file is empty", () => {
+      beforeEach(async () => {
+        await outputFile(join(qlPackPath, "qlpack.yml"), "", "utf-8");
+      });
+
+      it("should open query file in a new QL pack folder", async () => {
+        await wizard.execute();
+
+        expect(openTextDocumentSpy).toHaveBeenCalledWith(
+          Uri.joinPath(
+            Uri.file(qlPackPath),
+            "codeql-custom-queries-javascript",
+            "example.ql",
+          ),
+        );
+      });
+    });
+  });
+
   describe("findDatabaseItemByNwo", () => {
     describe("when the item exists", () => {
       it("should return the database item", async () => {
@@ -659,7 +811,9 @@ describe("SkeletonQueryWizard", () => {
         it("returns the parent path", async () => {
           const chosenPath = await wizard.determineStoragePath();
 
-          expect(chosenPath).toEqual(queriesDir.name);
+          expect(chosenPath).toEqual(
+            join(queriesDir.name, "codeql-custom-queries-swift"),
+          );
         });
       });
 

extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts

@@ -9,7 +9,6 @@ import * as tmp from "tmp";
 import { mockedObject } from "../utils/mocking.helpers";
 
 describe("QlPackGenerator", () => {
-  let packFolderName: string;
   let packFolderPath: string;
   let qlPackYamlFilePath: string;
   let exampleQlFilePath: string;
@@ -22,8 +21,9 @@ describe("QlPackGenerator", () => {
     dir = tmp.dirSync();
 
     language = "ruby";
-    packFolderName = `test-ql-pack-${language}`;
+    packFolderPath = Uri.file(
-    packFolderPath = Uri.file(join(dir.name, packFolderName)).fsPath;
+      join(dir.name, `test-ql-pack-${language}`),
+    ).fsPath;
 
     qlPackYamlFilePath = join(packFolderPath, "codeql-pack.yml");
     exampleQlFilePath = join(packFolderPath, "example.ql");
@@ -34,10 +34,9 @@ describe("QlPackGenerator", () => {
     });
 
     generator = new QlPackGenerator(
-      packFolderName,
       language as QueryLanguage,
       mockCli,
-      dir.name,
+      packFolderPath,
     );
   });