Merge pull request #2074 from github/elena/refactor-query-language

Consolidate the way we define query languages

extensions/ql-vscode/src/cli.ts

@@ -26,9 +26,9 @@ import { QueryMetadata, SortDirection } from "./pure/interface-types";
 import { Logger, ProgressReporter } from "./common";
 import { CompilationMessage } from "./pure/legacy-messages";
 import { sarifParser } from "./sarif-parser";
-import { dbSchemeToLanguage, walkDirectory } from "./helpers";
+import { walkDirectory } from "./helpers";
 import { App } from "./common/app";
-import { QueryLanguage } from "./qlpack-generator";
+import { QueryLanguage } from "./common/query-language";
 
 /**
  * The version of the SARIF format that we are using.
@@ -1179,9 +1179,11 @@ export class CodeQLCliServer implements Disposable {
    */
   public async getSupportedLanguages(): Promise<string[]> {
     if (!this._supportedLanguages) {
-      // Get the intersection of resolveLanguages with the list of hardcoded languages in dbSchemeToLanguage.
+      // Get the intersection of resolveLanguages with the list of languages in QueryLanguage.
       const resolvedLanguages = Object.keys(await this.resolveLanguages());
-      const hardcodedLanguages = Object.values(dbSchemeToLanguage);
+      const hardcodedLanguages = Object.values(QueryLanguage).map((s) =>
+        s.toString(),
+      );
 
       this._supportedLanguages = resolvedLanguages.filter((lang) =>
         hardcodedLanguages.includes(lang),

extensions/ql-vscode/src/common/query-language.ts

@@ -0,0 +1,37 @@
+export enum QueryLanguage {
+  CSharp = "csharp",
+  Cpp = "cpp",
+  Go = "go",
+  Java = "java",
+  Javascript = "javascript",
+  Python = "python",
+  Ruby = "ruby",
+  Swift = "swift",
+}
+
+export const PACKS_BY_QUERY_LANGUAGE = {
+  [QueryLanguage.Cpp]: ["codeql/cpp-queries"],
+  [QueryLanguage.CSharp]: [
+    "codeql/csharp-queries",
+    "codeql/csharp-solorigate-queries",
+  ],
+  [QueryLanguage.Go]: ["codeql/go-queries"],
+  [QueryLanguage.Java]: ["codeql/java-queries"],
+  [QueryLanguage.Javascript]: [
+    "codeql/javascript-queries",
+    "codeql/javascript-experimental-atm-queries",
+  ],
+  [QueryLanguage.Python]: ["codeql/python-queries"],
+  [QueryLanguage.Ruby]: ["codeql/ruby-queries"],
+};
+
+export const dbSchemeToLanguage = {
+  "semmlecode.javascript.dbscheme": "javascript",
+  "semmlecode.cpp.dbscheme": "cpp",
+  "semmlecode.dbscheme": "java",
+  "semmlecode.python.dbscheme": "python",
+  "semmlecode.csharp.dbscheme": "csharp",
+  "go.dbscheme": "go",
+  "ruby.dbscheme": "ruby",
+  "swift.dbscheme": "swift",
+};

extensions/ql-vscode/src/helpers.ts

@@ -24,6 +24,7 @@ import { QueryMetadata } from "./pure/interface-types";
 import { telemetryListener } from "./telemetry";
 import { RedactableError } from "./pure/errors";
 import { getQlPackPath } from "./pure/ql";
+import { dbSchemeToLanguage } from "./common/query-language";
 
 // Shared temporary folder for the extension.
 export const tmpDir = dirSync({
@@ -568,16 +569,6 @@ export class CachedOperation<U> {
  *
  * @see cli.CodeQLCliServer.resolveDatabase
  */
-export const dbSchemeToLanguage = {
-  "semmlecode.javascript.dbscheme": "javascript",
-  "semmlecode.cpp.dbscheme": "cpp",
-  "semmlecode.dbscheme": "java",
-  "semmlecode.python.dbscheme": "python",
-  "semmlecode.csharp.dbscheme": "csharp",
-  "go.dbscheme": "go",
-  "ruby.dbscheme": "ruby",
-  "swift.dbscheme": "swift",
-};
 
 export const languageToDbScheme = Object.entries(dbSchemeToLanguage).reduce(
   (acc, [k, v]) => {

extensions/ql-vscode/src/local-databases.ts

@@ -26,7 +26,8 @@ import { QueryRunner } from "./queryRunner";
 import { pathsEqual } from "./pure/files";
 import { redactableError } from "./pure/errors";
 import { isCodespacesTemplate } from "./config";
-import { QlPackGenerator, QueryLanguage } from "./qlpack-generator";
+import { QlPackGenerator } from "./qlpack-generator";
+import { QueryLanguage } from "./common/query-language";
 
 /**
  * databases.ts

extensions/ql-vscode/src/packaging.ts

@@ -9,18 +9,7 @@ import { ProgressCallback, UserCancellationException } from "./commandRunner";
 import { extLogger } from "./common";
 import { asError, getErrorStack } from "./pure/helpers-pure";
 import { redactableError } from "./pure/errors";
-
-const QUERY_PACKS = [
-  "codeql/cpp-queries",
-  "codeql/csharp-queries",
-  "codeql/go-queries",
-  "codeql/java-queries",
-  "codeql/javascript-queries",
-  "codeql/python-queries",
-  "codeql/ruby-queries",
-  "codeql/csharp-solorigate-queries",
-  "codeql/javascript-experimental-atm-queries",
-];
+import { PACKS_BY_QUERY_LANGUAGE } from "./common/query-language";
 
 /**
  * Prompts user to choose packs to download, and downloads them.
@@ -45,7 +34,7 @@ export async function handleDownloadPacks(
     { ignoreFocusOut: true },
   );
   if (quickpick === queryPackOption) {
-    packsToDownload = QUERY_PACKS;
+    packsToDownload = Object.values(PACKS_BY_QUERY_LANGUAGE).flat();
   } else if (quickpick === customPackOption) {
     const customPack = await window.showInputBox({
       prompt:

extensions/ql-vscode/src/qlpack-generator.ts

@@ -3,16 +3,7 @@ import { dump } from "js-yaml";
 import { join } from "path";
 import { Uri, workspace } from "vscode";
 import { CodeQLCliServer } from "./cli";
-
-export type QueryLanguage =
-  | "csharp"
-  | "cpp"
-  | "go"
-  | "java"
-  | "javascript"
-  | "python"
-  | "ruby"
-  | "swift";
+import { QueryLanguage } from "./common/query-language";
 
 export class QlPackGenerator {
   private readonly qlpackName: string;

extensions/ql-vscode/src/variant-analysis/gh-api/variant-analysis.ts

@@ -1,28 +1,19 @@
+import { QueryLanguage } from "../../common/query-language";
 import { Repository, RepositoryWithMetadata } from "./repository";
 
 export interface VariantAnalysisSubmissionRequest {
   action_repo_ref: string;
-  language: VariantAnalysisQueryLanguage;
+  language: QueryLanguage;
   query_pack: string;
   repositories?: string[];
   repository_lists?: string[];
   repository_owners?: string[];
 }
 
-export type VariantAnalysisQueryLanguage =
-  | "csharp"
-  | "cpp"
-  | "go"
-  | "java"
-  | "javascript"
-  | "python"
-  | "ruby"
-  | "swift";
-
 export interface VariantAnalysis {
   id: number;
   controller_repo: Repository;
-  query_language: VariantAnalysisQueryLanguage;
+  query_language: QueryLanguage;
   query_pack_url: string;
   created_at: string;
   updated_at: string;

extensions/ql-vscode/src/variant-analysis/shared/variant-analysis.ts

@@ -1,5 +1,6 @@
 import { Repository, RepositoryWithMetadata } from "./repository";
 import { AnalysisAlert, AnalysisRawResults } from "./analysis-result";
+import { QueryLanguage } from "../../common/query-language";
 
 export interface VariantAnalysis {
   id: number;
@@ -7,7 +8,7 @@ export interface VariantAnalysis {
   query: {
     name: string;
     filePath: string;
-    language: VariantAnalysisQueryLanguage;
+    language: QueryLanguage;
     text: string;
   };
   databases: {
@@ -26,23 +27,10 @@ export interface VariantAnalysis {
   skippedRepos?: VariantAnalysisSkippedRepositories;
 }
 
-export enum VariantAnalysisQueryLanguage {
-  CSharp = "csharp",
-  Cpp = "cpp",
-  Go = "go",
-  Java = "java",
-  Javascript = "javascript",
-  Python = "python",
-  Ruby = "ruby",
-  Swift = "swift",
-}
-
 export function parseVariantAnalysisQueryLanguage(
   language: string,
-): VariantAnalysisQueryLanguage | undefined {
-  return Object.values(VariantAnalysisQueryLanguage).find(
-    (x) => x === language,
-  );
+): QueryLanguage | undefined {
+  return Object.values(QueryLanguage).find((x) => x === language);
 }
 
 export enum VariantAnalysisStatus {
@@ -148,7 +136,7 @@ export interface VariantAnalysisSubmission {
   query: {
     name: string;
     filePath: string;
-    language: VariantAnalysisQueryLanguage;
+    language: QueryLanguage;
     text: string;
 
     // Base64 encoded query pack.

extensions/ql-vscode/test/factories/variant-analysis/gh-api/variant-analysis-api-response.ts

@@ -5,7 +5,7 @@ import {
   VariantAnalysisSkippedRepositories,
   VariantAnalysisStatus,
 } from "../../../../src/variant-analysis/gh-api/variant-analysis";
-import { VariantAnalysisQueryLanguage } from "../../../../src/variant-analysis/shared/variant-analysis";
+import { QueryLanguage } from "../../../../src/common/query-language";
 import { createMockScannedRepos } from "./scanned-repositories";
 import { createMockSkippedRepos } from "./skipped-repositories";
 import { createMockRepository } from "./repository";
@@ -23,7 +23,7 @@ export function createMockApiResponse(
       full_name: "github/pickles",
       private: false,
     },
-    query_language: VariantAnalysisQueryLanguage.Javascript,
+    query_language: QueryLanguage.Javascript,
     query_pack_url: "https://example.com/foo",
     created_at: faker.date.recent().toISOString(),
     updated_at: faker.date.recent().toISOString(),

extensions/ql-vscode/test/factories/variant-analysis/shared/variant-analysis-submission.ts

@@ -1,8 +1,6 @@
 import { faker } from "@faker-js/faker";
-import {
-  VariantAnalysisQueryLanguage,
-  VariantAnalysisSubmission,
-} from "../../../../src/variant-analysis/shared/variant-analysis";
+import { VariantAnalysisSubmission } from "../../../../src/variant-analysis/shared/variant-analysis";
+import { QueryLanguage } from "../../../../src/common/query-language";
 
 export function createMockSubmission(): VariantAnalysisSubmission {
   return {
@@ -12,7 +10,7 @@ export function createMockSubmission(): VariantAnalysisSubmission {
     query: {
       name: "query-name",
       filePath: "query-file-path",
-      language: VariantAnalysisQueryLanguage.Javascript,
+      language: QueryLanguage.Javascript,
       text: "query-text",
       pack: "base64-encoded-string",
     },

extensions/ql-vscode/test/factories/variant-analysis/shared/variant-analysis.ts

@@ -1,7 +1,6 @@
 import { faker } from "@faker-js/faker";
 import {
   VariantAnalysis,
-  VariantAnalysisQueryLanguage,
   VariantAnalysisScannedRepository,
   VariantAnalysisSkippedRepositories,
   VariantAnalysisStatus,
@@ -9,6 +8,7 @@ import {
 import { createMockScannedRepos } from "./scanned-repositories";
 import { createMockSkippedRepos } from "./skipped-repositories";
 import { createMockRepository } from "./repository";
+import { QueryLanguage } from "../../../../src/common/query-language";
 
 export function createMockVariantAnalysis({
   status = VariantAnalysisStatus.InProgress,
@@ -32,7 +32,7 @@ export function createMockVariantAnalysis({
     query: {
       name: "a-query-name",
       filePath: "a-query-file-path",
-      language: VariantAnalysisQueryLanguage.Javascript,
+      language: QueryLanguage.Javascript,
       text: "a-query-text",
     },
     databases: {

extensions/ql-vscode/test/unit-tests/databases/db-tree-creator.test.ts

@@ -16,6 +16,7 @@ import {
   createLocalTree,
   createRemoteTree,
 } from "../../../src/databases/db-tree-creator";
+import { QueryLanguage } from "../../../src/common/query-language";
 import { createDbConfig } from "../../factories/db-config-factories";
 
 describe("db tree creator", () => {
@@ -334,7 +335,7 @@ describe("db tree creator", () => {
               {
                 name: "db1",
                 dateAdded: 1668428293677,
-                language: "cpp",
+                language: QueryLanguage.Cpp,
                 storagePath: "/path/to/db1/",
               },
               {

extensions/ql-vscode/test/unit-tests/variant-analysis.test.ts

@@ -1,7 +1,6 @@
 import {
   VariantAnalysis,
   parseVariantAnalysisQueryLanguage,
-  VariantAnalysisQueryLanguage,
   VariantAnalysisStatus,
   isVariantAnalysisComplete,
   VariantAnalysisRepoStatus,
@@ -9,11 +8,12 @@ import {
 } from "../../src/variant-analysis/shared/variant-analysis";
 import { createMockScannedRepo } from "../factories/variant-analysis/shared/scanned-repositories";
 import { createMockVariantAnalysis } from "../factories/variant-analysis/shared/variant-analysis";
+import { QueryLanguage } from "../../src/common/query-language";
 
 describe("parseVariantAnalysisQueryLanguage", () => {
   it("parses a valid language", () => {
     expect(parseVariantAnalysisQueryLanguage("javascript")).toBe(
-      VariantAnalysisQueryLanguage.Javascript,
+      QueryLanguage.Javascript,
     );
   });
 

extensions/ql-vscode/test/unit-tests/variant-analysis/markdown-generation.test.ts

@@ -6,11 +6,11 @@ import {
   MarkdownFile,
 } from "../../../src/variant-analysis/markdown-generation";
 import {
-  VariantAnalysisQueryLanguage,
   VariantAnalysisRepoStatus,
   VariantAnalysisScannedRepository,
   VariantAnalysisScannedRepositoryResult,
 } from "../../../src/variant-analysis/shared/variant-analysis";
+import { QueryLanguage } from "../../../src/common/query-language";
 import {
   AnalysisAlert,
   AnalysisRawResults,
@@ -32,7 +32,7 @@ describe(generateVariantAnalysisMarkdown.name, () => {
             filePath:
               "c:\\git-repo\\vscode-codeql-starter\\ql\\javascript\\ql\\src\\Security\\CWE-078\\ShellCommandInjectionFromEnvironment.ql",
             text: '/**\n * @name Shell command built from environment values\n * @description Building a shell command string with values from the enclosing\n *              environment may cause subtle bugs or vulnerabilities.\n * @kind path-problem\n * @problem.severity warning\n * @security-severity 6.3\n * @precision high\n * @id js/shell-command-injection-from-environment\n * @tags correctness\n *       security\n *       external/cwe/cwe-078\n *       external/cwe/cwe-088\n */\n\nimport javascript\nimport DataFlow::PathGraph\nimport semmle.javascript.security.dataflow.ShellCommandInjectionFromEnvironmentQuery\n\nfrom\n  Configuration cfg, DataFlow::PathNode source, DataFlow::PathNode sink, DataFlow::Node highlight,\n  Source sourceNode\nwhere\n  sourceNode = source.getNode() and\n  cfg.hasFlowPath(source, sink) and\n  if cfg.isSinkWithHighlight(sink.getNode(), _)\n  then cfg.isSinkWithHighlight(sink.getNode(), highlight)\n  else highlight = sink.getNode()\nselect highlight, source, sink, "This shell command depends on an uncontrolled $@.", sourceNode,\n  sourceNode.getSourceType()\n',
-            language: VariantAnalysisQueryLanguage.Javascript,
+            language: QueryLanguage.Javascript,
           },
         },
         getResults(pathProblemAnalysesResults),
@@ -56,7 +56,7 @@ describe(generateVariantAnalysisMarkdown.name, () => {
             filePath:
               "c:\\git-repo\\vscode-codeql-starter\\ql\\javascript\\ql\\src\\Performance\\ReDoS.ql",
             text: '/**\n * @name Inefficient regular expression\n * @description A regular expression that requires exponential time to match certain inputs\n *              can be a performance bottleneck, and may be vulnerable to denial-of-service\n *              attacks.\n * @kind problem\n * @problem.severity error\n * @security-severity 7.5\n * @precision high\n * @id js/redos\n * @tags security\n *       external/cwe/cwe-1333\n *       external/cwe/cwe-730\n *       external/cwe/cwe-400\n */\n\nimport javascript\nimport semmle.javascript.security.performance.ReDoSUtil\nimport semmle.javascript.security.performance.ExponentialBackTracking\n\nfrom RegExpTerm t, string pump, State s, string prefixMsg\nwhere hasReDoSResult(t, pump, s, prefixMsg)\nselect t,\n  "This part of the regular expression may cause exponential backtracking on strings " + prefixMsg +\n    "containing many repetitions of \'" + pump + "\'."\n',
-            language: VariantAnalysisQueryLanguage.Javascript,
+            language: QueryLanguage.Javascript,
           },
         },
         getResults(problemAnalysesResults),
@@ -79,7 +79,7 @@ describe(generateVariantAnalysisMarkdown.name, () => {
             name: "Contradictory guard nodes",
             filePath: "c:\\Users\\foo\\bar\\quick-query.ql",
             text: '/**\n * @name Contradictory guard nodes\n * \n * @description Snippet from "UselessComparisonTest.ql"\n */\n\nimport javascript\n\n/**\n * Holds if there are any contradictory guard nodes in `container`.\n *\n * We use this to restrict reachability analysis to a small set of containers.\n */\npredicate hasContradictoryGuardNodes(StmtContainer container) {\n  exists(ConditionGuardNode guard |\n    RangeAnalysis::isContradictoryGuardNode(guard) and\n    container = guard.getContainer()\n  )\n}\n\nfrom StmtContainer c\nwhere hasContradictoryGuardNodes(c)\nselect c, c.getNumLines()',
-            language: VariantAnalysisQueryLanguage.Javascript,
+            language: QueryLanguage.Javascript,
           },
         },
         getResults(rawResultsAnalysesResults),

extensions/ql-vscode/test/unit-tests/variant-analysis/variant-analysis-processor.test.ts

@@ -1,7 +1,6 @@
 import { faker } from "@faker-js/faker";
 import { VariantAnalysisScannedRepository as ApiVariantAnalysisScannedRepository } from "../../../src/variant-analysis/gh-api/variant-analysis";
 import {
-  VariantAnalysisQueryLanguage,
   VariantAnalysisScannedRepository,
   VariantAnalysisRepoStatus,
 } from "../../../src/variant-analysis/shared/variant-analysis";
@@ -18,6 +17,7 @@ import { createMockSkippedRepos } from "../../factories/variant-analysis/gh-api/
 import { createMockApiResponse } from "../../factories/variant-analysis/gh-api/variant-analysis-api-response";
 import { createMockSubmission } from "../../factories/variant-analysis/shared/variant-analysis-submission";
 import { createMockVariantAnalysisRepoTask } from "../../factories/variant-analysis/gh-api/variant-analysis-repo-task";
+import { QueryLanguage } from "../../../src/common/query-language";
 
 describe(processVariantAnalysis.name, () => {
   const scannedRepos = createMockScannedRepos();
@@ -48,7 +48,7 @@ describe(processVariantAnalysis.name, () => {
       },
       query: {
         filePath: "query-file-path",
-        language: VariantAnalysisQueryLanguage.Javascript,
+        language: QueryLanguage.Javascript,
         name: "query-name",
         text: mockSubmission.query.text,
       },

extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts

@@ -14,6 +14,7 @@ import { ExtensionApp } from "../../../../src/common/vscode/vscode-app";
 import { createMockExtensionContext } from "../../../factories/extension-context";
 import { createDbConfig } from "../../../factories/db-config-factories";
 import { setRemoteControllerRepo } from "../../../../src/config";
+import { QueryLanguage } from "../../../../src/common/query-language";
 
 describe("db panel rendering nodes", () => {
   const workspaceStoragePath = join(__dirname, "test-workspace-storage");
@@ -183,13 +184,13 @@ describe("db panel rendering nodes", () => {
               {
                 name: "db1",
                 dateAdded: 1668428293677,
-                language: "cpp",
+                language: QueryLanguage.Cpp,
                 storagePath: "/path/to/db1/",
               },
               {
                 name: "db2",
                 dateAdded: 1668428472731,
-                language: "cpp",
+                language: QueryLanguage.Cpp,
                 storagePath: "/path/to/db2/",
               },
             ],
@@ -234,7 +235,7 @@ describe("db panel rendering nodes", () => {
           kind: DbItemKind.LocalDatabase,
           databaseName: "db1",
           dateAdded: 1668428293677,
-          language: "cpp",
+          language: QueryLanguage.Cpp,
           storagePath: "/path/to/db1/",
           selected: false,
         },
@@ -242,7 +243,7 @@ describe("db panel rendering nodes", () => {
           kind: DbItemKind.LocalDatabase,
           databaseName: "db2",
           dateAdded: 1668428472731,
-          language: "cpp",
+          language: QueryLanguage.Cpp,
           storagePath: "/path/to/db2/",
           selected: false,
         },

extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts

@@ -1,6 +1,7 @@
 import { join } from "path";
 import { existsSync } from "fs";
-import { QlPackGenerator, QueryLanguage } from "../../../src/qlpack-generator";
+import { QlPackGenerator } from "../../../src/qlpack-generator";
+import { QueryLanguage } from "../../../src/common/query-language";
 import { CodeQLCliServer } from "../../../src/cli";
 import { Uri, workspace } from "vscode";
 import { getErrorMessage } from "../../../src/pure/helpers-pure";

extensions/ql-vscode/test/vscode-tests/no-workspace/helpers.test.ts

@@ -30,6 +30,7 @@ import {
   walkDirectory,
 } from "../../../src/helpers";
 import { reportStreamProgress } from "../../../src/commandRunner";
+import { QueryLanguage } from "../../../src/common/query-language";
 
 describe("helpers", () => {
   describe("Invocation rate limiter", () => {
@@ -146,10 +147,14 @@ describe("helpers", () => {
 
   describe("codeql-database.yml tests", () => {
     let dir: tmp.DirResult;
+    let language: QueryLanguage;
+
     beforeEach(() => {
       dir = tmp.dirSync();
+      language = QueryLanguage.Cpp;
+
       const contents = dump({
-        primaryLanguage: "cpp",
+        primaryLanguage: language,
       });
       writeFileSync(join(dir.name, "codeql-database.yml"), contents, "utf8");
     });
@@ -159,7 +164,7 @@ describe("helpers", () => {
     });
 
     it("should get initial query contents when language is known", () => {
-      expect(getInitialQueryContents("cpp", "hucairz")).toBe(
+      expect(getInitialQueryContents(language, "hucairz")).toBe(
         'import cpp\n\nselect ""',
       );
     });

extensions/ql-vscode/test/vscode-tests/no-workspace/local-databases-ui.test.ts

@@ -12,6 +12,7 @@ import { Uri } from "vscode";
 import { DatabaseUI } from "../../../src/local-databases-ui";
 import { testDisposeHandler } from "../test-dispose-handler";
 import { createMockApp } from "../../__mocks__/appMock";
+import { QueryLanguage } from "../../../src/common/query-language";
 
 describe("local-databases-ui", () => {
   describe("fixDbUri", () => {
@@ -64,21 +65,25 @@ describe("local-databases-ui", () => {
 
   it("should delete orphaned databases", async () => {
     const storageDir = dirSync().name;
-    const db1 = createDatabase(storageDir, "db1-imported", "cpp");
-    const db2 = createDatabase(storageDir, "db2-notimported", "cpp");
+    const db1 = createDatabase(storageDir, "db1-imported", QueryLanguage.Cpp);
+    const db2 = createDatabase(
+      storageDir,
+      "db2-notimported",
+      QueryLanguage.Cpp,
+    );
     const db3 = createDatabase(storageDir, "db3-invalidlanguage", "hucairz");
 
     // these two should be deleted
     const db4 = createDatabase(
       storageDir,
       "db2-notimported-with-db-info",
-      "cpp",
+      QueryLanguage.Cpp,
       ".dbinfo",
     );
     const db5 = createDatabase(
       storageDir,
       "db2-notimported-with-codeql-database.yml",
-      "cpp",
+      QueryLanguage.Cpp,
       "codeql-database.yml",
     );