mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Avoiding overwriting any existing modeled methods from the files

Browse Source
This commit is contained in:
Robert
2023-07-18 15:04:58 +01:00
parent db4dc89e42
commit 5b170d02eb
4 changed files with 548 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts
View File

@@ -123,6 +123,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
msg.externalApiUsages,
msg.modeledMethods,
this.mode,
this.cliServer,
this.app.logger,
);
await Promise.all([this.setViewState(), this.loadExternalApiUsages()]);

16
extensions/ql-vscode/src/data-extensions-editor/modeled-method-fs.ts
View File

@@ -5,11 +5,7 @@ import { Mode } from "./shared/mode";
import { createDataExtensionYamls, loadDataExtensionYaml } from "./yaml";
import { join, relative } from "path";
import { ExtensionPack } from "./shared/extension-pack";
import {
Logger,
NotificationLogger,
showAndLogErrorMessage,
} from "../common/logging";
import { NotificationLogger, showAndLogErrorMessage } from "../common/logging";
import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
import { load as loadYaml } from "js-yaml";
import { CodeQLCliServer } from "../codeql-cli/cli";
@@ -22,13 +18,21 @@ export async function saveModeledMethods(
externalApiUsages: ExternalApiUsage[],
modeledMethods: Record<string, ModeledMethod>,
mode: Mode,
logger: Logger,
cliServer: CodeQLCliServer,
logger: NotificationLogger,
): Promise<void> {
const existingModeledMethods = await loadModeledMethodFiles(
extensionPack,
cliServer,
logger,
);
const yamls = createDataExtensionYamls(
databaseName,
language,
externalApiUsages,
modeledMethods,
existingModeledMethods,
mode,
);

79
extensions/ql-vscode/src/data-extensions-editor/yaml.ts
View File

@@ -72,7 +72,8 @@ export function createDataExtensionYamls(
databaseName: string,
language: string,
externalApiUsages: ExternalApiUsage[],
modeledMethods: Record<string, ModeledMethod>,
newModeledMethods: Record<string, ModeledMethod>,
existingModeledMethods: Record<string, Record<string, ModeledMethod>>,
mode: Mode,
) {
switch (mode) {
@@ -80,14 +81,16 @@ export function createDataExtensionYamls(
return createDataExtensionYamlsForApplicationMode(
language,
externalApiUsages,
modeledMethods,
newModeledMethods,
existingModeledMethods,
);
case Mode.Framework:
return createDataExtensionYamlsForFrameworkMode(
databaseName,
language,
externalApiUsages,
modeledMethods,
newModeledMethods,
existingModeledMethods,
);
default:
assertNever(mode);
@@ -97,27 +100,51 @@ export function createDataExtensionYamls(
export function createDataExtensionYamlsForApplicationMode(
language: string,
externalApiUsages: ExternalApiUsage[],
modeledMethods: Record<string, ModeledMethod>,
newModeledMethods: Record<string, ModeledMethod>,
existingModeledMethods: Record<string, Record<string, ModeledMethod>>,
): Record<string, string> {
const methodsByLibraryFilename: Record<string, ModeledMethod[]> = {};
const methodsByLibraryFilename: Record<
string,
Record<string, ModeledMethod>
> = {};
// We only want to generate a yaml file when it's a known external API usage
// and there are new modeled methods for it. This avoids us overwriting other
// files that may contain data we don't know about.
for (const externalApiUsage of externalApiUsages) {
const modeledMethod = modeledMethods[externalApiUsage.signature];
if (!modeledMethod) {
continue;
if (externalApiUsage.signature in newModeledMethods) {
methodsByLibraryFilename[
createFilenameForLibrary(externalApiUsage.library)
] = {};
}
}
const filename = createFilenameForLibrary(externalApiUsage.library);
// First populate methodsByLibraryFilename with any existing modeled methods.
for (const [filename, methods] of Object.entries(existingModeledMethods)) {
if (filename in methodsByLibraryFilename) {
for (const [signature, method] of Object.entries(methods)) {
methodsByLibraryFilename[filename][signature] = method;
}
}
}
methodsByLibraryFilename[filename] =
methodsByLibraryFilename[filename] || [];
methodsByLibraryFilename[filename].push(modeledMethod);
// Add the new modeled methods, potentially overwriting existing modeled methods
// but not removing existing modeled methods that are not in the new set.
for (const externalApiUsage of externalApiUsages) {
const method = newModeledMethods[externalApiUsage.signature];
if (method) {
const filename = createFilenameForLibrary(externalApiUsage.library);
methodsByLibraryFilename[filename][method.signature] = method;
}
}
const result: Record<string, string> = {};
for (const [filename, methods] of Object.entries(methodsByLibraryFilename)) {
result[filename] = createDataExtensionYaml(language, methods);
result[filename] = createDataExtensionYaml(
language,
Object.values(methods),
);
}
return result;
@@ -127,7 +154,8 @@ export function createDataExtensionYamlsForFrameworkMode(
databaseName: string,
language: string,
externalApiUsages: ExternalApiUsage[],
modeledMethods: Record<string, ModeledMethod>,
newModeledMethods: Record<string, ModeledMethod>,
existingModeledMethods: Record<string, Record<string, ModeledMethod>>,
prefix = "models/",
suffix = ".model",
): Record<string, string> {
@@ -136,16 +164,25 @@ export function createDataExtensionYamlsForFrameworkMode(
.slice(1)
.map((part) => sanitizeExtensionPackName(part))
.join("-");
const filename = `${prefix}${libraryName}${suffix}.yml`;
const methods = externalApiUsages
.map((externalApiUsage) => modeledMethods[externalApiUsage.signature])
.filter((modeledMethod) => modeledMethod !== undefined);
const methods: Record<string, ModeledMethod> = {};
for (const [signature, method] of Object.entries(
existingModeledMethods[filename] || {},
)) {
methods[signature] = method;
}
for (const externalApiUsage of externalApiUsages) {
const modeledMethod = newModeledMethods[externalApiUsage.signature];
if (modeledMethod) {
methods[modeledMethod.signature] = modeledMethod;
}
}
return {
[`${prefix}${libraryName}${suffix}.yml`]: createDataExtensionYaml(
language,
methods,
),
[filename]: createDataExtensionYaml(language, Object.values(methods)),
};
}

481
extensions/ql-vscode/test/unit-tests/data-extensions-editor/yaml.test.ts
View File

@@ -76,7 +76,7 @@ describe("createDataExtensionYaml", () => {
});
describe("createDataExtensionYamlsForApplicationMode", () => {
it("creates the correct YAML files", () => {
it("creates the correct YAML files when there are no existing modeled methods", () => {
const yaml = createDataExtensionYamlsForApplicationMode(
"java",
[
@@ -262,6 +262,7 @@ describe("createDataExtensionYamlsForApplicationMode", () => {
methodParameters: "(String,String,String)",
},
},
{},
);
expect(yaml).toEqual({
@@ -312,10 +313,290 @@ describe("createDataExtensionYamlsForApplicationMode", () => {
`,
});
});
it("creates the correct YAML files when there are existing modeled methods", () => {
const yaml = createDataExtensionYamlsForApplicationMode(
"java",
[
{
library: "sql2o",
libraryVersion: "1.6.0",
signature: "org.sql2o.Connection#createQuery(String)",
packageName: "org.sql2o",
typeName: "Connection",
methodName: "createQuery",
methodParameters: "(String)",
supported: true,
supportedType: "sink",
usages: [
{
label: "createQuery(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 15,
startColumn: 13,
endLine: 15,
endColumn: 56,
},
classification: CallClassification.Source,
},
{
label: "createQuery(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 26,
startColumn: 13,
endLine: 26,
endColumn: 39,
},
classification: CallClassification.Source,
},
],
},
{
library: "sql2o",
libraryVersion: "1.6.0",
signature: "org.sql2o.Query#executeScalar(Class)",
packageName: "org.sql2o",
typeName: "Query",
methodName: "executeScalar",
methodParameters: "(Class)",
supported: true,
supportedType: "neutral",
usages: [
{
label: "executeScalar(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 15,
startColumn: 13,
endLine: 15,
endColumn: 85,
},
classification: CallClassification.Source,
},
{
label: "executeScalar(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 26,
startColumn: 13,
endLine: 26,
endColumn: 68,
},
classification: CallClassification.Source,
},
],
},
{
library: "sql2o",
libraryVersion: "2.5.0-alpha1",
signature: "org.sql2o.Sql2o#Sql2o(String,String,String)",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "Sql2o",
methodParameters: "(String,String,String)",
supported: false,
supportedType: "none",
usages: [
{
label: "new Sql2o(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 10,
startColumn: 33,
endLine: 10,
endColumn: 88,
},
classification: CallClassification.Source,
},
],
},
{
library: "spring-boot",
libraryVersion: "3.0.2",
signature:
"org.springframework.boot.SpringApplication#run(Class,String[])",
packageName: "org.springframework.boot",
typeName: "SpringApplication",
methodName: "run",
methodParameters: "(Class,String[])",
supported: false,
supportedType: "none",
usages: [
{
label: "run(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/Sql2oExampleApplication.java",
startLine: 9,
startColumn: 9,
endLine: 9,
endColumn: 66,
},
classification: CallClassification.Source,
},
],
},
{
library: "rt",
signature: "java.io.PrintStream#println(String)",
packageName: "java.io",
typeName: "PrintStream",
methodName: "println",
methodParameters: "(String)",
supported: true,
supportedType: "summary",
usages: [
{
label: "println(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 29,
startColumn: 9,
endLine: 29,
endColumn: 49,
},
classification: CallClassification.Source,
},
],
},
],
{
"org.sql2o.Connection#createQuery(String)": {
type: "sink",
input: "Argument[0]",
output: "",
kind: "sql",
provenance: "df-generated",
signature: "org.sql2o.Connection#createQuery(String)",
packageName: "org.sql2o",
typeName: "Connection",
methodName: "createQuery",
methodParameters: "(String)",
},
"org.springframework.boot.SpringApplication#run(Class,String[])": {
type: "neutral",
input: "",
output: "",
kind: "summary",
provenance: "manual",
signature:
"org.springframework.boot.SpringApplication#run(Class,String[])",
packageName: "org.springframework.boot",
typeName: "SpringApplication",
methodName: "run",
methodParameters: "(Class,String[])",
},
"org.sql2o.Sql2o#Sql2o(String,String,String)": {
type: "sink",
input: "Argument[0]",
output: "",
kind: "jndi",
provenance: "manual",
signature: "org.sql2o.Sql2o#Sql2o(String,String,String)",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "Sql2o",
methodParameters: "(String,String,String)",
},
},
{
"models/sql2o.model.yml": {
"org.sql2o.Connection#createQuery(String)": {
type: "neutral",
input: "",
output: "",
kind: "summary",
provenance: "manual",
signature: "org.sql2o.Connection#createQuery(String)",
packageName: "org.sql2o",
typeName: "Connection",
methodName: "createQuery",
methodParameters: "(String)",
},
"org.sql2o.Query#executeScalar(Class)": {
type: "neutral",
input: "",
output: "",
kind: "summary",
provenance: "manual",
signature: "org.sql2o.Query#executeScalar(Class)",
packageName: "org.sql2o",
typeName: "Query",
methodName: "executeScalar",
methodParameters: "(Class)",
},
},
"models/gson.model.yml": {
"com.google.gson.TypeAdapter#fromJsonTree(JsonElement)": {
type: "summary",
input: "Argument[this]",
output: "ReturnValue",
kind: "taint",
provenance: "df-generated",
signature: "com.google.gson.TypeAdapter#fromJsonTree(JsonElement)",
packageName: "com.google.gson",
typeName: "TypeAdapter",
methodName: "fromJsonTree",
methodParameters: "(JsonElement)",
},
},
},
);
expect(yaml).toEqual({
"models/sql2o.model.yml": `extensions:
- addsTo:
pack: codeql/java-all
extensible: sourceModel
data: []
- addsTo:
pack: codeql/java-all
extensible: sinkModel
data:
- ["org.sql2o","Connection",true,"createQuery","(String)","","Argument[0]","sql","df-generated"]
- ["org.sql2o","Sql2o",true,"Sql2o","(String,String,String)","","Argument[0]","jndi","manual"]
- addsTo:
pack: codeql/java-all
extensible: summaryModel
data: []
- addsTo:
pack: codeql/java-all
extensible: neutralModel
data:
- ["org.sql2o","Query","executeScalar","(Class)","summary","manual"]
`,
"models/spring-boot.model.yml": `extensions:
- addsTo:
pack: codeql/java-all
extensible: sourceModel
data: []
- addsTo:
pack: codeql/java-all
extensible: sinkModel
data: []
- addsTo:
pack: codeql/java-all
extensible: summaryModel
data: []
- addsTo:
pack: codeql/java-all
extensible: neutralModel
data:
- ["org.springframework.boot","SpringApplication","run","(Class,String[])","summary","manual"]
`,
});
});
});
describe("createDataExtensionYamlsForFrameworkMode", () => {
it("creates the correct YAML files", () => {
it("creates the correct YAML files when there are no existing modeled methods", () => {
const yaml = createDataExtensionYamlsForFrameworkMode(
"github/sql2o",
"java",
@@ -438,6 +719,7 @@ describe("createDataExtensionYamlsForFrameworkMode", () => {
methodParameters: "(String,String,String)",
},
},
{},
);
expect(yaml).toEqual({
@@ -463,6 +745,201 @@ describe("createDataExtensionYamlsForFrameworkMode", () => {
pack: codeql/java-all
extensible: neutralModel
data: []
`,
});
});
it("creates the correct YAML files when there are existing modeled methods", () => {
const yaml = createDataExtensionYamlsForFrameworkMode(
"github/sql2o",
"java",
[
{
library: "sql2o",
signature: "org.sql2o.Connection#createQuery(String)",
packageName: "org.sql2o",
typeName: "Connection",
methodName: "createQuery",
methodParameters: "(String)",
supported: true,
supportedType: "sink",
usages: [
{
label: "createQuery(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 15,
startColumn: 13,
endLine: 15,
endColumn: 56,
},
classification: CallClassification.Source,
},
{
label: "createQuery(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 26,
startColumn: 13,
endLine: 26,
endColumn: 39,
},
classification: CallClassification.Source,
},
],
},
{
library: "sql2o",
signature: "org.sql2o.Query#executeScalar(Class)",
packageName: "org.sql2o",
typeName: "Query",
methodName: "executeScalar",
methodParameters: "(Class)",
supported: true,
supportedType: "neutral",
usages: [
{
label: "executeScalar(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 15,
startColumn: 13,
endLine: 15,
endColumn: 85,
},
classification: CallClassification.Source,
},
{
label: "executeScalar(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 26,
startColumn: 13,
endLine: 26,
endColumn: 68,
},
classification: CallClassification.Source,
},
],
},
{
library: "sql2o",
signature: "org.sql2o.Sql2o#Sql2o(String,String,String)",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "Sql2o",
methodParameters: "(String,String,String)",
supported: false,
supportedType: "none",
usages: [
{
label: "new Sql2o(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 10,
startColumn: 33,
endLine: 10,
endColumn: 88,
},
classification: CallClassification.Source,
},
],
},
],
{
"org.sql2o.Connection#createQuery(String)": {
type: "sink",
input: "Argument[0]",
output: "",
kind: "sql",
provenance: "df-generated",
signature: "org.sql2o.Connection#createQuery(String)",
packageName: "org.sql2o",
typeName: "Connection",
methodName: "createQuery",
methodParameters: "(String)",
},
"org.sql2o.Sql2o#Sql2o(String,String,String)": {
type: "sink",
input: "Argument[0]",
output: "",
kind: "jndi",
provenance: "manual",
signature: "org.sql2o.Sql2o#Sql2o(String,String,String)",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "Sql2o",
methodParameters: "(String,String,String)",
},
},
{
"models/sql2o.model.yml": {
"org.sql2o.Connection#createQuery(String)": {
type: "neutral",
input: "",
output: "",
kind: "summary",
provenance: "manual",
signature: "org.sql2o.Connection#createQuery(String)",
packageName: "org.sql2o",
typeName: "Connection",
methodName: "createQuery",
methodParameters: "(String)",
},
"org.sql2o.Query#executeScalar(Class)": {
type: "neutral",
input: "",
output: "",
kind: "summary",
provenance: "manual",
signature: "org.sql2o.Query#executeScalar(Class)",
packageName: "org.sql2o",
typeName: "Query",
methodName: "executeScalar",
methodParameters: "(Class)",
},
},
"models/gson.model.yml": {
"com.google.gson.TypeAdapter#fromJsonTree(JsonElement)": {
type: "summary",
input: "Argument[this]",
output: "ReturnValue",
kind: "taint",
provenance: "df-generated",
signature: "com.google.gson.TypeAdapter#fromJsonTree(JsonElement)",
packageName: "com.google.gson",
typeName: "TypeAdapter",
methodName: "fromJsonTree",
methodParameters: "(JsonElement)",
},
},
},
);
expect(yaml).toEqual({
"models/sql2o.model.yml": `extensions:
- addsTo:
pack: codeql/java-all
extensible: sourceModel
data: []
- addsTo:
pack: codeql/java-all
extensible: sinkModel
data:
- ["org.sql2o","Connection",true,"createQuery","(String)","","Argument[0]","sql","df-generated"]
- ["org.sql2o","Sql2o",true,"Sql2o","(String,String,String)","","Argument[0]","jndi","manual"]
- addsTo:
pack: codeql/java-all
extensible: summaryModel
data: []
- addsTo:
pack: codeql/java-all
extensible: neutralModel
data:
- ["org.sql2o","Query","executeScalar","(Class)","summary","manual"]
`,
});
});