mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #3028 from github/koesie10/bqrs-decode-languages

Browse Source 
Use MaD definition when decoding BQRS
This commit is contained in:
Koen Vlaswinkel
2023-10-27 11:58:48 +02:00
committed by GitHub
parent 14c6f98289 ecb2503992
commit f4a2d8572c
11 changed files with 189 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
extensions/ql-vscode/src/model-editor/bqrs.ts
View File

@@ -4,13 +4,18 @@ import { ModeledMethodType } from "./modeled-method";
import { parseLibraryFilename } from "./library";
import { Mode } from "./shared/mode";
import { ApplicationModeTuple, FrameworkModeTuple } from "./queries/query";
import { QueryLanguage } from "../common/query-language";
import { getModelsAsDataLanguage } from "./languages";
export function decodeBqrsToMethods(
chunk: DecodedBqrsChunk,
mode: Mode,
language: QueryLanguage,
): Method[] {
const methodsByApiName = new Map<string, Method>();
const definition = getModelsAsDataLanguage(language);
chunk?.tuples.forEach((tuple) => {
let usage: Call;
let packageName: string;
@@ -51,7 +56,12 @@ export function decodeBqrsToMethods(
classification = CallClassification.Unknown;
}
const signature = `${packageName}.${typeName}#${methodName}${methodParameters}`;
const signature = definition.createMethodSignature({
packageName,
typeName,
methodName,
methodParameters,
});
// For Java, we'll always get back a .jar file, and the library version may be bad because not all library authors
// properly specify the version. Therefore, we'll always try to parse the name and version from the library filename

2
extensions/ql-vscode/src/model-editor/flow-model-queries.ts
View File

@@ -156,7 +156,7 @@ async function runSingleFlowQuery(
// Interpret the results
const modelsAsDataLanguage = getModelsAsDataLanguage(language);
const definition = modelsAsDataLanguage[type];
const definition = modelsAsDataLanguage.predicates[type];
const bqrsPath = completedQuery.outputDir.bqrsPath;

12
extensions/ql-vscode/src/model-editor/languages/models-as-data.ts
View File

@@ -1,3 +1,4 @@
import { MethodDefinition } from "../method";
import { ModeledMethod, ModeledMethodType } from "../modeled-method";
import { DataTuple } from "../model-extension-file";
@@ -6,14 +7,19 @@ type ReadModeledMethod = (row: DataTuple[]) => ModeledMethod;
export type ModelsAsDataLanguageModelType = Exclude<ModeledMethodType, "none">;
export type ModelsAsDataLanguageModel = {
export type ModelsAsDataLanguagePredicate = {
extensiblePredicate: string;
supportedKinds: string[];
generateMethodDefinition: GenerateMethodDefinition;
readModeledMethod: ReadModeledMethod;
};
export type ModelsAsDataLanguage = Record<
export type ModelsAsDataLanguagePredicates = Record<
ModelsAsDataLanguageModelType,
ModelsAsDataLanguageModel
ModelsAsDataLanguagePredicate
>;
export type ModelsAsDataLanguage = {
createMethodSignature: (method: MethodDefinition) => string;
predicates: ModelsAsDataLanguagePredicates;
};

248
extensions/ql-vscode/src/model-editor/languages/static.ts
View File

@@ -8,125 +8,133 @@ function readRowToMethod(row: DataTuple[]): string {
}
export const staticLanguage: ModelsAsDataLanguage = {
source: {
extensiblePredicate: sharedExtensiblePredicates.source,
supportedKinds: sharedKinds.source,
// extensible predicate sourceModel(
// string package, string type, boolean subtypes, string name, string signature, string ext,
// string output, string kind, string provenance
// );
generateMethodDefinition: (method) => [
method.packageName,
method.typeName,
true,
method.methodName,
method.methodParameters,
"",
method.output,
method.kind,
method.provenance,
],
readModeledMethod: (row) => ({
type: "source" as ModeledMethodType,
input: "",
output: row[6] as string,
kind: row[7] as string,
provenance: row[8] as Provenance,
signature: readRowToMethod(row),
packageName: row[0] as string,
typeName: row[1] as string,
methodName: row[3] as string,
methodParameters: row[4] as string,
}),
},
sink: {
extensiblePredicate: sharedExtensiblePredicates.sink,
supportedKinds: sharedKinds.sink,
// extensible predicate sinkModel(
// string package, string type, boolean subtypes, string name, string signature, string ext,
// string input, string kind, string provenance
// );
generateMethodDefinition: (method) => [
method.packageName,
method.typeName,
true,
method.methodName,
method.methodParameters,
"",
method.input,
method.kind,
method.provenance,
],
readModeledMethod: (row) => ({
type: "sink",
input: row[6] as string,
output: "",
kind: row[7] as string,
provenance: row[8] as Provenance,
signature: readRowToMethod(row),
packageName: row[0] as string,
typeName: row[1] as string,
methodName: row[3] as string,
methodParameters: row[4] as string,
}),
},
summary: {
extensiblePredicate: sharedExtensiblePredicates.summary,
supportedKinds: sharedKinds.summary,
// extensible predicate summaryModel(
// string package, string type, boolean subtypes, string name, string signature, string ext,
// string input, string output, string kind, string provenance
// );
generateMethodDefinition: (method) => [
method.packageName,
method.typeName,
true,
method.methodName,
method.methodParameters,
"",
method.input,
method.output,
method.kind,
method.provenance,
],
readModeledMethod: (row) => ({
type: "summary",
input: row[6] as string,
output: row[7] as string,
kind: row[8] as string,
provenance: row[9] as Provenance,
signature: readRowToMethod(row),
packageName: row[0] as string,
typeName: row[1] as string,
methodName: row[3] as string,
methodParameters: row[4] as string,
}),
},
neutral: {
extensiblePredicate: sharedExtensiblePredicates.neutral,
supportedKinds: sharedKinds.neutral,
// extensible predicate neutralModel(
// string package, string type, string name, string signature, string kind, string provenance
// );
generateMethodDefinition: (method) => [
method.packageName,
method.typeName,
method.methodName,
method.methodParameters,
method.kind,
method.provenance,
],
readModeledMethod: (row) => ({
type: "neutral",
input: "",
output: "",
kind: row[4] as string,
provenance: row[5] as Provenance,
signature: `${row[0]}.${row[1]}#${row[2]}${row[3]}`,
packageName: row[0] as string,
typeName: row[1] as string,
methodName: row[2] as string,
methodParameters: row[3] as string,
}),
createMethodSignature: ({
packageName,
typeName,
methodName,
methodParameters,
}) => `${packageName}.${typeName}#${methodName}${methodParameters}`,
predicates: {
source: {
extensiblePredicate: sharedExtensiblePredicates.source,
supportedKinds: sharedKinds.source,
// extensible predicate sourceModel(
// string package, string type, boolean subtypes, string name, string signature, string ext,
// string output, string kind, string provenance
// );
generateMethodDefinition: (method) => [
method.packageName,
method.typeName,
true,
method.methodName,
method.methodParameters,
"",
method.output,
method.kind,
method.provenance,
],
readModeledMethod: (row) => ({
type: "source" as ModeledMethodType,
input: "",
output: row[6] as string,
kind: row[7] as string,
provenance: row[8] as Provenance,
signature: readRowToMethod(row),
packageName: row[0] as string,
typeName: row[1] as string,
methodName: row[3] as string,
methodParameters: row[4] as string,
}),
},
sink: {
extensiblePredicate: sharedExtensiblePredicates.sink,
supportedKinds: sharedKinds.sink,
// extensible predicate sinkModel(
// string package, string type, boolean subtypes, string name, string signature, string ext,
// string input, string kind, string provenance
// );
generateMethodDefinition: (method) => [
method.packageName,
method.typeName,
true,
method.methodName,
method.methodParameters,
"",
method.input,
method.kind,
method.provenance,
],
readModeledMethod: (row) => ({
type: "sink",
input: row[6] as string,
output: "",
kind: row[7] as string,
provenance: row[8] as Provenance,
signature: readRowToMethod(row),
packageName: row[0] as string,
typeName: row[1] as string,
methodName: row[3] as string,
methodParameters: row[4] as string,
}),
},
summary: {
extensiblePredicate: sharedExtensiblePredicates.summary,
supportedKinds: sharedKinds.summary,
// extensible predicate summaryModel(
// string package, string type, boolean subtypes, string name, string signature, string ext,
// string input, string output, string kind, string provenance
// );
generateMethodDefinition: (method) => [
method.packageName,
method.typeName,
true,
method.methodName,
method.methodParameters,
"",
method.input,
method.output,
method.kind,
method.provenance,
],
readModeledMethod: (row) => ({
type: "summary",
input: row[6] as string,
output: row[7] as string,
kind: row[8] as string,
provenance: row[9] as Provenance,
signature: readRowToMethod(row),
packageName: row[0] as string,
typeName: row[1] as string,
methodName: row[3] as string,
methodParameters: row[4] as string,
}),
},
neutral: {
extensiblePredicate: sharedExtensiblePredicates.neutral,
supportedKinds: sharedKinds.neutral,
// extensible predicate neutralModel(
// string package, string type, string name, string signature, string kind, string provenance
// );
generateMethodDefinition: (method) => [
method.packageName,
method.typeName,
method.methodName,
method.methodParameters,
method.kind,
method.provenance,
],
readModeledMethod: (row) => ({
type: "neutral",
input: "",
output: "",
kind: row[4] as string,
provenance: row[5] as Provenance,
signature: `${row[0]}.${row[1]}#${row[2]}${row[3]}`,
packageName: row[0] as string,
typeName: row[1] as string,
methodName: row[2] as string,
methodParameters: row[3] as string,
}),
},
},
};

29
extensions/ql-vscode/src/model-editor/method.ts
View File

@@ -17,19 +17,7 @@ export type Usage = Call & {
readonly classification: CallClassification;
};
export interface MethodSignature {
/**
* Contains the version of the library if it can be determined by CodeQL, e.g. `4.2.2.2`
*/
readonly libraryVersion?: string;
/**
* A unique signature that can be used to identify this external API usage.
*
* The signature contains the package name, type name, method name, and method parameters
* in the form "packageName.typeName#methodName(methodParameters)".
* e.g. `org.sql2o.Connection#createQuery(String)`
*/
readonly signature: string;
export interface MethodDefinition {
/**
* The package name in Java, or the namespace in C#, e.g. `org.sql2o` or `System.Net.Http.Headers`.
*
@@ -44,6 +32,21 @@ export interface MethodSignature {
readonly methodParameters: string;
}
export interface MethodSignature extends MethodDefinition {
/**
* Contains the version of the library if it can be determined by CodeQL, e.g. `4.2.2.2`
*/
readonly libraryVersion?: string;
/**
* A unique signature that can be used to identify this external API usage.
*
* The signature contains the package name, type name, method name, and method parameters
* in the form "packageName.typeName#methodName(methodParameters)".
* e.g. `org.sql2o.Connection#createQuery(String)`
*/
readonly signature: string;
}
export interface Method extends MethodSignature {
/**
* Contains the name of the library containing the method declaration, e.g. `sql2o-1.6.0.jar` or `System.Runtime.dll`

4
extensions/ql-vscode/src/model-editor/model-editor-queries.ts
View File

@@ -25,6 +25,7 @@ type RunQueryOptions = {
cliServer: CodeQLCliServer;
queryRunner: QueryRunner;
databaseItem: DatabaseItem;
language: QueryLanguage;
queryStorageDir: string;
queryDir: string;
@@ -70,6 +71,7 @@ export async function runModelEditorQueries(
cliServer,
queryRunner,
databaseItem,
language,
queryStorageDir,
queryDir,
progress,
@@ -157,7 +159,7 @@ export async function runModelEditorQueries(
maxStep: externalApiQueriesProgressMaxStep,
});
return decodeBqrsToMethods(bqrsChunk, mode);
return decodeBqrsToMethods(bqrsChunk, mode, language);
}
type GetResultsOptions = {

1
extensions/ql-vscode/src/model-editor/model-editor-view.ts
View File

@@ -430,6 +430,7 @@ export class ModelEditorView extends AbstractWebview<
cliServer: this.cliServer,
queryRunner: this.queryRunner,
databaseItem: this.databaseItem,
language: this.language,
queryStorageDir: this.queryStorageDir,
queryDir: this.queryDir,
progress: (update) =>

8
extensions/ql-vscode/src/model-editor/yaml.ts
View File

@@ -4,7 +4,7 @@ import { Method } from "./method";
import { ModeledMethod, ModeledMethodType } from "./modeled-method";
import {
getModelsAsDataLanguage,
ModelsAsDataLanguageModel,
ModelsAsDataLanguagePredicate,
} from "./languages";
import * as modelExtensionFileSchema from "./model-extension-file.schema.json";
@@ -18,7 +18,7 @@ const modelExtensionFileSchemaValidate = ajv.compile(modelExtensionFileSchema);
function createDataProperty(
methods: readonly ModeledMethod[],
definition: ModelsAsDataLanguageModel,
definition: ModelsAsDataLanguagePredicate,
) {
if (methods.length === 0) {
return " []";
@@ -56,7 +56,7 @@ export function createDataExtensionYaml(
}
}
const extensions = Object.entries(modelsAsDataLanguage).map(
const extensions = Object.entries(modelsAsDataLanguage.predicates).map(
([type, definition]) => ` - addsTo:
pack: codeql/${language}-all
extensible: ${definition.extensiblePredicate}
@@ -260,7 +260,7 @@ export function loadDataExtensionYaml(
const extensible = addsTo.extensible;
const data = extension.data;
const definition = Object.values(modelsAsDataLanguage).find(
const definition = Object.values(modelsAsDataLanguage.predicates).find(
(definition) => definition.extensiblePredicate === extensible,
);
if (!definition) {

2
extensions/ql-vscode/src/view/model-editor/ModelKindDropdown.tsx
View File

@@ -26,7 +26,7 @@ export const ModelKindDropdown = ({
const modelsAsDataLanguage = getModelsAsDataLanguage(language);
return modeledMethod?.type && modeledMethod.type !== "none"
? modelsAsDataLanguage[modeledMethod.type]
? modelsAsDataLanguage.predicates[modeledMethod.type]
: undefined;
}, [language, modeledMethod?.type]);

17
extensions/ql-vscode/test/unit-tests/model-editor/bqrs.test.ts
View File

@@ -2,6 +2,7 @@ import { decodeBqrsToMethods } from "../../../src/model-editor/bqrs";
import { DecodedBqrsChunk } from "../../../src/common/bqrs-cli-types";
import { CallClassification } from "../../../src/model-editor/method";
import { Mode } from "../../../src/model-editor/shared/mode";
import { QueryLanguage } from "../../../src/common/query-language";
describe("decodeBqrsToMethods", () => {
describe("Java queries", () => {
@@ -237,7 +238,9 @@ describe("decodeBqrsToMethods", () => {
// Even though there are a number of methods with the same number of usages, the order returned should be stable:
// - Iterating over a map (as done by .values()) is guaranteed to be in insertion order
// - Sorting the array of methods is guaranteed to be a stable sort
expect(decodeBqrsToMethods(chunk, Mode.Application)).toEqual([
expect(
decodeBqrsToMethods(chunk, Mode.Application, QueryLanguage.Java),
).toEqual([
{
library: "rt",
libraryVersion: undefined,
@@ -499,7 +502,9 @@ describe("decodeBqrsToMethods", () => {
};
it("extracts methods", () => {
expect(decodeBqrsToMethods(chunk, Mode.Framework)).toEqual([
expect(
decodeBqrsToMethods(chunk, Mode.Framework, QueryLanguage.Java),
).toEqual([
{
library: "",
libraryVersion: undefined,
@@ -594,7 +599,9 @@ describe("decodeBqrsToMethods", () => {
};
it("extracts methods", () => {
expect(decodeBqrsToMethods(chunk, Mode.Application)).toEqual([
expect(
decodeBqrsToMethods(chunk, Mode.Application, QueryLanguage.Java),
).toEqual([
{
library: "mscorlib",
libraryVersion: "4.0.0.0",
@@ -660,7 +667,9 @@ describe("decodeBqrsToMethods", () => {
};
it("extracts methods", () => {
expect(decodeBqrsToMethods(chunk, Mode.Framework)).toEqual([
expect(
decodeBqrsToMethods(chunk, Mode.Framework, QueryLanguage.Java),
).toEqual([
{
library: "Times",
libraryVersion: undefined,

2
extensions/ql-vscode/test/vscode-tests/no-workspace/model-editor/external-api-usage-query.test.ts
View File

@@ -74,6 +74,7 @@ describe("runModelEditorQueries", () => {
},
language,
}),
language,
queryStorageDir: "/tmp/queries",
queryDir,
progress: jest.fn(),
@@ -139,6 +140,7 @@ describe("runModelEditorQueries", () => {
},
language,
}),
language,
queryStorageDir: "/tmp/queries",
queryDir,
progress: jest.fn(),