Merge pull request #2329 from github/starcke/ext-csharp-query

Add support for running C# query.

extensions/ql-vscode/src/data-extensions-editor/queries/csharp.ts

@@ -0,0 +1,198 @@
+import { Query } from "./query";
+
+export const fetchExternalApisQuery: Query = {
+  mainQuery: `/**
+ * @name Usage of APIs coming from external libraries
+ * @description A list of 3rd party APIs used in the codebase.
+ * @tags telemetry
+ * @id cs/telemetry/fetch-external-apis
+ */
+
+ import csharp
+ import semmle.code.csharp.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
+ import ExternalApi
+ 
+ private Call aUsage(ExternalApi api) {
+   result.getTarget().getUnboundDeclaration() = api
+ }
+ 
+ private boolean isSupported(ExternalApi api) {
+   api.isSupported() and result = true
+   or
+   not api.isSupported() and
+   result = false
+ }
+ 
+ from ExternalApi api, string apiName, boolean supported, Call usage
+ where
+   apiName = api.getApiName() and
+   supported = isSupported(api) and
+   usage = aUsage(api)
+ select apiName, supported, usage
+`,
+  dependencies: {
+    "ExternalApi.qll": `/** Provides classes and predicates related to handling APIs from external libraries. */
+
+private import csharp
+private import dotnet
+private import semmle.code.csharp.dispatch.Dispatch
+private import semmle.code.csharp.dataflow.ExternalFlow
+private import semmle.code.csharp.dataflow.FlowSummary
+private import semmle.code.csharp.dataflow.internal.DataFlowImplCommon as DataFlowImplCommon
+private import semmle.code.csharp.dataflow.internal.DataFlowPrivate
+private import semmle.code.csharp.dataflow.internal.DataFlowDispatch as DataFlowDispatch
+private import semmle.code.csharp.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
+private import semmle.code.csharp.dataflow.internal.TaintTrackingPrivate
+private import semmle.code.csharp.security.dataflow.flowsources.Remote
+
+pragma[nomagic]
+private predicate isTestNamespace(Namespace ns) {
+  ns.getFullName()
+      .matches([
+          "NUnit.Framework%", "Xunit%", "Microsoft.VisualStudio.TestTools.UnitTesting%", "Moq%"
+        ])
+}
+
+/**
+ * A test library.
+ */
+class TestLibrary extends RefType {
+  TestLibrary() { isTestNamespace(this.getNamespace()) }
+}
+
+/** Holds if the given callable is not worth supporting. */
+private predicate isUninteresting(DotNet::Callable c) {
+  c.getDeclaringType() instanceof TestLibrary or
+  c.(Constructor).isParameterless()
+}
+
+/**
+ * An external API from either the C# Standard Library or a 3rd party library.
+ */
+class ExternalApi extends DotNet::Callable {
+  ExternalApi() {
+    this.isUnboundDeclaration() and
+    this.fromLibrary() and
+    this.(Modifiable).isEffectivelyPublic() and
+    not isUninteresting(this)
+  }
+
+  /**
+   * Gets the unbound type, name and parameter types of this API.
+   */
+  bindingset[this]
+  private string getSignature() {
+    result =
+      this.getDeclaringType().getUnboundDeclaration() + "." + this.getName() + "(" +
+        parameterQualifiedTypeNamesToString(this) + ")"
+  }
+
+  /**
+   * Gets the namespace of this API.
+   */
+  bindingset[this]
+  string getNamespace() { this.getDeclaringType().hasQualifiedName(result, _) }
+
+  /**
+   * Gets the namespace and signature of this API.
+   */
+  bindingset[this]
+  string getApiName() { result = this.getNamespace() + "#" + this.getSignature() }
+
+  /** Gets a node that is an input to a call to this API. */
+  private ArgumentNode getAnInput() {
+    result
+        .getCall()
+        .(DataFlowDispatch::NonDelegateDataFlowCall)
+        .getATarget(_)
+        .getUnboundDeclaration() = this
+  }
+
+  /** Gets a node that is an output from a call to this API. */
+  private DataFlow::Node getAnOutput() {
+    exists(
+      Call c, DataFlowDispatch::NonDelegateDataFlowCall dc, DataFlowImplCommon::ReturnKindExt ret
+    |
+      dc.getDispatchCall().getCall() = c and
+      c.getTarget().getUnboundDeclaration() = this
+    |
+      result = ret.getAnOutNode(dc)
+    )
+  }
+
+  /** Holds if this API has a supported summary. */
+  pragma[nomagic]
+  predicate hasSummary() {
+    this instanceof SummarizedCallable
+    or
+    defaultAdditionalTaintStep(this.getAnInput(), _)
+  }
+
+  /** Holds if this API is a known source. */
+  pragma[nomagic]
+  predicate isSource() {
+    this.getAnOutput() instanceof RemoteFlowSource or sourceNode(this.getAnOutput(), _)
+  }
+
+  /** Holds if this API is a known sink. */
+  pragma[nomagic]
+  predicate isSink() { sinkNode(this.getAnInput(), _) }
+
+  /** Holds if this API is a known neutral. */
+  pragma[nomagic]
+  predicate isNeutral() { this instanceof FlowSummaryImpl::Public::NeutralCallable }
+
+  /**
+   * Holds if this API is supported by existing CodeQL libraries, that is, it is either a
+   * recognized source, sink or neutral or it has a flow summary.
+   */
+  predicate isSupported() {
+    this.hasSummary() or this.isSource() or this.isSink() or this.isNeutral()
+  }
+}
+
+/**
+ * Gets the limit for the number of results produced by a telemetry query.
+ */
+int resultLimit() { result = 1000 }
+
+/**
+ * Holds if it is relevant to count usages of "api".
+ */
+signature predicate relevantApi(ExternalApi api);
+
+/**
+ * Given a predicate to count relevant API usages, this module provides a predicate
+ * for restricting the number or returned results based on a certain limit.
+ */
+module Results<relevantApi/1 getRelevantUsages> {
+  private int getUsages(string apiName) {
+    result =
+      strictcount(Call c, ExternalApi api |
+        c.getTarget().getUnboundDeclaration() = api and
+        apiName = api.getApiName() and
+        getRelevantUsages(api)
+      )
+  }
+
+  private int getOrder(string apiName) {
+    apiName =
+      rank[result](string name, int usages |
+        usages = getUsages(name)
+      |
+        name order by usages desc, name
+      )
+  }
+
+  /**
+   * Holds if there exists an API with "apiName" that is being used "usages" times
+   * and if it is in the top results (guarded by resultLimit).
+   */
+  predicate restrict(string apiName, int usages) {
+    usages = getUsages(apiName) and
+    getOrder(apiName) <= resultLimit()
+  }
+}
+`,
+  },
+};

extensions/ql-vscode/src/data-extensions-editor/queries/index.ts

@@ -1,7 +1,9 @@
+import { fetchExternalApisQuery as csharpFetchExternalApisQuery } from "./csharp";
 import { fetchExternalApisQuery as javaFetchExternalApisQuery } from "./java";
 import { Query } from "./query";
 import { QueryLanguage } from "../../common/query-language";
 
 export const fetchExternalApiQueries: Partial<Record<QueryLanguage, Query>> = {
+  [QueryLanguage.CSharp]: csharpFetchExternalApisQuery,
   [QueryLanguage.Java]: javaFetchExternalApisQuery,
 };

extensions/ql-vscode/src/data-extensions-editor/queries/java.ts

@@ -20,11 +20,7 @@ private Call aUsage(ExternalApi api) {
 private boolean isSupported(ExternalApi api) {
   api.isSupported() and result = true
   or
-  api = any(FlowSummaryImpl::Public::NeutralCallable nsc).asCallable() and result = true
-  or
-  not api.isSupported() and
-  not api = any(FlowSummaryImpl::Public::NeutralCallable nsc).asCallable() and
-  result = false
+  not api.isSupported() and result = false
 }
 
 from ExternalApi api, string apiName, boolean supported, Call usage

extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts

@@ -5,13 +5,12 @@ import {
 import { createMockLogger } from "../../../__mocks__/loggerMock";
 import type { Uri } from "vscode";
 import { DatabaseKind } from "../../../../src/local-databases";
-import * as queryResolver from "../../../../src/contextual/queryResolver";
 import { file } from "tmp-promise";
 import { QueryResultType } from "../../../../src/pure/new-messages";
 import { readdir, readFile } from "fs-extra";
 import { load } from "js-yaml";
 import { dirname, join } from "path";
-import { fetchExternalApisQuery } from "../../../../src/data-extensions-editor/queries/java";
+import { fetchExternalApiQueries } from "../../../../src/data-extensions-editor/queries/index";
 import * as helpers from "../../../../src/helpers";
 import { RedactableError } from "../../../../src/pure/errors";
 
@@ -29,100 +28,97 @@ function createMockUri(path = "/a/b/c/foo"): Uri {
 }
 
 describe("runQuery", () => {
-  it("runs the query", async () => {
-    jest.spyOn(queryResolver, "qlpackOfDatabase").mockResolvedValue({
-      dbschemePack: "codeql/java-all",
-      dbschemePackIsLibraryPack: false,
-      queryPack: "codeql/java-queries",
-    });
-
+  it("runs all queries", async () => {
     const logPath = (await file()).path;
 
-    const options = {
-      cliServer: {
-        resolveQlpacks: jest.fn().mockResolvedValue({
-          "my/java-extensions": "/a/b/c/",
-        }),
-      },
-      queryRunner: {
-        createQueryRun: jest.fn().mockReturnValue({
-          evaluate: jest.fn().mockResolvedValue({
-            resultType: QueryResultType.SUCCESS,
+    // Test all queries
+    for (const [lang, query] of Object.entries(fetchExternalApiQueries)) {
+      const options = {
+        cliServer: {
+          resolveQlpacks: jest.fn().mockResolvedValue({
+            "my/extensions": "/a/b/c/",
           }),
-          outputDir: {
-            logPath,
-          },
-        }),
-        logger: createMockLogger(),
-      },
-      databaseItem: {
-        databaseUri: createMockUri("/a/b/c/src.zip"),
-        contents: {
-          kind: DatabaseKind.Database,
-          name: "foo",
-          datasetUri: createMockUri(),
         },
-        language: "java",
-      },
-      queryStorageDir: "/tmp/queries",
-      progress: jest.fn(),
-      token: {
-        isCancellationRequested: false,
-        onCancellationRequested: jest.fn(),
-      },
-    };
-    const result = await runQuery(options);
+        queryRunner: {
+          createQueryRun: jest.fn().mockReturnValue({
+            evaluate: jest.fn().mockResolvedValue({
+              resultType: QueryResultType.SUCCESS,
+            }),
+            outputDir: {
+              logPath,
+            },
+          }),
+          logger: createMockLogger(),
+        },
+        databaseItem: {
+          databaseUri: createMockUri("/a/b/c/src.zip"),
+          contents: {
+            kind: DatabaseKind.Database,
+            name: "foo",
+            datasetUri: createMockUri(),
+          },
+          language: lang,
+        },
+        queryStorageDir: "/tmp/queries",
+        progress: jest.fn(),
+        token: {
+          isCancellationRequested: false,
+          onCancellationRequested: jest.fn(),
+        },
+      };
+      const result = await runQuery(options);
 
-    expect(result?.resultType).toEqual(QueryResultType.SUCCESS);
+      expect(result?.resultType).toEqual(QueryResultType.SUCCESS);
 
-    expect(options.cliServer.resolveQlpacks).toHaveBeenCalledTimes(1);
-    expect(options.cliServer.resolveQlpacks).toHaveBeenCalledWith([], true);
-    expect(options.queryRunner.createQueryRun).toHaveBeenCalledWith(
-      "/a/b/c/src.zip",
-      {
-        queryPath: expect.stringMatching(/FetchExternalApis\.ql/),
-        quickEvalPosition: undefined,
-      },
-      false,
-      [],
-      ["my/java-extensions"],
-      "/tmp/queries",
-      undefined,
-      undefined,
-    );
-
-    const queryPath =
-      options.queryRunner.createQueryRun.mock.calls[0][1].queryPath;
-    const queryDirectory = dirname(queryPath);
-
-    const queryFiles = await readdir(queryDirectory);
-    expect(queryFiles.sort()).toEqual(
-      ["codeql-pack.yml", "FetchExternalApis.ql", "ExternalApi.qll"].sort(),
-    );
-
-    const suiteFileContents = await readFile(
-      join(queryDirectory, "codeql-pack.yml"),
-      "utf8",
-    );
-    const suiteYaml = load(suiteFileContents);
-    expect(suiteYaml).toEqual({
-      name: "codeql/external-api-usage",
-      version: "0.0.0",
-      dependencies: {
-        "codeql/java-all": "*",
-      },
-    });
-
-    expect(
-      await readFile(join(queryDirectory, "FetchExternalApis.ql"), "utf8"),
-    ).toEqual(fetchExternalApisQuery.mainQuery);
-
-    for (const [filename, contents] of Object.entries(
-      fetchExternalApisQuery.dependencies ?? {},
-    )) {
-      expect(await readFile(join(queryDirectory, filename), "utf8")).toEqual(
-        contents,
+      expect(options.cliServer.resolveQlpacks).toHaveBeenCalledTimes(1);
+      expect(options.cliServer.resolveQlpacks).toHaveBeenCalledWith([], true);
+      expect(options.queryRunner.createQueryRun).toHaveBeenCalledWith(
+        "/a/b/c/src.zip",
+        {
+          queryPath: expect.stringMatching(/FetchExternalApis\.ql/),
+          quickEvalPosition: undefined,
+        },
+        false,
+        [],
+        ["my/extensions"],
+        "/tmp/queries",
+        undefined,
+        undefined,
       );
+
+      const queryPath =
+        options.queryRunner.createQueryRun.mock.calls[0][1].queryPath;
+      const queryDirectory = dirname(queryPath);
+
+      const queryFiles = await readdir(queryDirectory);
+      expect(queryFiles.sort()).toEqual(
+        ["codeql-pack.yml", "FetchExternalApis.ql", "ExternalApi.qll"].sort(),
+      );
+
+      const suiteFileContents = await readFile(
+        join(queryDirectory, "codeql-pack.yml"),
+        "utf8",
+      );
+      const suiteYaml = load(suiteFileContents);
+      expect(suiteYaml).toEqual({
+        name: "codeql/external-api-usage",
+        version: "0.0.0",
+        dependencies: {
+          [`codeql/${lang}-all`]: "*",
+        },
+      });
+
+      expect(
+        await readFile(join(queryDirectory, "FetchExternalApis.ql"), "utf8"),
+      ).toEqual(query.mainQuery);
+
+      for (const [filename, contents] of Object.entries(
+        query.dependencies ?? {},
+      )) {
+        expect(await readFile(join(queryDirectory, filename), "utf8")).toEqual(
+          contents,
+        );
+      }
     }
   });
 });