mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #2621 from github/robertbrignull/data-saving

Browse Source 
Load existing modeled methods before saving, to avoid overwriting data
This commit is contained in:
Robert
2023-07-20 08:54:29 +01:00
committed by GitHub
parent f2c7c41117 bb246144c2
commit d8687b5985
5 changed files with 581 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts
View File

@@ -123,6 +123,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
msg.externalApiUsages,
msg.modeledMethods,
this.mode,
this.cliServer,
this.app.logger,
);
await Promise.all([this.setViewState(), this.loadExternalApiUsages()]);

49
extensions/ql-vscode/src/data-extensions-editor/modeled-method-fs.ts
View File

@@ -3,13 +3,9 @@ import { ExternalApiUsage } from "./external-api-usage";
import { ModeledMethod } from "./modeled-method";
import { Mode } from "./shared/mode";
import { createDataExtensionYamls, loadDataExtensionYaml } from "./yaml";
import { join } from "path";
import { join, relative } from "path";
import { ExtensionPack } from "./shared/extension-pack";
import {
Logger,
NotificationLogger,
showAndLogErrorMessage,
} from "../common/logging";
import { NotificationLogger, showAndLogErrorMessage } from "../common/logging";
import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
import { load as loadYaml } from "js-yaml";
import { CodeQLCliServer } from "../codeql-cli/cli";
@@ -22,13 +18,21 @@ export async function saveModeledMethods(
externalApiUsages: ExternalApiUsage[],
modeledMethods: Record<string, ModeledMethod>,
mode: Mode,
logger: Logger,
cliServer: CodeQLCliServer,
logger: NotificationLogger,
): Promise<void> {
const existingModeledMethods = await loadModeledMethodFiles(
extensionPack,
cliServer,
logger,
);
const yamls = createDataExtensionYamls(
databaseName,
language,
externalApiUsages,
modeledMethods,
existingModeledMethods,
mode,
);
@@ -39,17 +43,20 @@ export async function saveModeledMethods(
void logger.log(`Saved data extension YAML`);
}
export async function loadModeledMethods(
async function loadModeledMethodFiles(
extensionPack: ExtensionPack,
cliServer: CodeQLCliServer,
logger: NotificationLogger,
): Promise<Record<string, ModeledMethod>> {
): Promise<Record<string, Record<string, ModeledMethod>>> {
const modelFiles = await listModelFiles(extensionPack.path, cliServer);
const existingModeledMethods: Record<string, ModeledMethod> = {};
const modeledMethodsByFile: Record<
string,
Record<string, ModeledMethod>
> = {};
for (const modelFile of modelFiles) {
const yaml = await readFile(modelFile, "utf8");
const yaml = await readFile(join(extensionPack.path, modelFile), "utf8");
const data = loadYaml(yaml, {
filename: modelFile,
@@ -63,7 +70,25 @@ export async function loadModeledMethods(
);
continue;
}
modeledMethodsByFile[modelFile] = modeledMethods;
}
return modeledMethodsByFile;
}
export async function loadModeledMethods(
extensionPack: ExtensionPack,
cliServer: CodeQLCliServer,
logger: NotificationLogger,
): Promise<Record<string, ModeledMethod>> {
const existingModeledMethods: Record<string, ModeledMethod> = {};
const modeledMethodsByFile = await loadModeledMethodFiles(
extensionPack,
cliServer,
logger,
);
for (const modeledMethods of Object.values(modeledMethodsByFile)) {
for (const [key, value] of Object.entries(modeledMethods)) {
existingModeledMethods[key] = value;
}
@@ -85,7 +110,7 @@ export async function listModelFiles(
for (const [path, extensions] of Object.entries(result.data)) {
if (pathsEqual(path, extensionPackPath)) {
for (const extension of extensions) {
modelFiles.add(extension.file);
modelFiles.add(relative(extensionPackPath, extension.file));
}
}
}

82
extensions/ql-vscode/src/data-extensions-editor/yaml.ts
View File

@@ -72,7 +72,8 @@ export function createDataExtensionYamls(
databaseName: string,
language: string,
externalApiUsages: ExternalApiUsage[],
modeledMethods: Record<string, ModeledMethod>,
newModeledMethods: Record<string, ModeledMethod>,
existingModeledMethods: Record<string, Record<string, ModeledMethod>>,
mode: Mode,
) {
switch (mode) {
@@ -80,14 +81,16 @@ export function createDataExtensionYamls(
return createDataExtensionYamlsForApplicationMode(
language,
externalApiUsages,
modeledMethods,
newModeledMethods,
existingModeledMethods,
);
case Mode.Framework:
return createDataExtensionYamlsForFrameworkMode(
databaseName,
language,
externalApiUsages,
modeledMethods,
newModeledMethods,
existingModeledMethods,
);
default:
assertNever(mode);
@@ -97,27 +100,51 @@ export function createDataExtensionYamls(
export function createDataExtensionYamlsForApplicationMode(
language: string,
externalApiUsages: ExternalApiUsage[],
modeledMethods: Record<string, ModeledMethod>,
newModeledMethods: Record<string, ModeledMethod>,
existingModeledMethods: Record<string, Record<string, ModeledMethod>>,
): Record<string, string> {
const methodsByLibraryFilename: Record<string, ModeledMethod[]> = {};
const methodsByLibraryFilename: Record<
string,
Record<string, ModeledMethod>
> = {};
// We only want to generate a yaml file when it's a known external API usage
// and there are new modeled methods for it. This avoids us overwriting other
// files that may contain data we don't know about.
for (const externalApiUsage of externalApiUsages) {
const modeledMethod = modeledMethods[externalApiUsage.signature];
if (!modeledMethod) {
continue;
if (externalApiUsage.signature in newModeledMethods) {
methodsByLibraryFilename[
createFilenameForLibrary(externalApiUsage.library)
] = {};
}
}
const filename = createFilenameForLibrary(externalApiUsage.library);
// First populate methodsByLibraryFilename with any existing modeled methods.
for (const [filename, methods] of Object.entries(existingModeledMethods)) {
if (filename in methodsByLibraryFilename) {
for (const [signature, method] of Object.entries(methods)) {
methodsByLibraryFilename[filename][signature] = method;
}
}
}
methodsByLibraryFilename[filename] =
methodsByLibraryFilename[filename] || [];
methodsByLibraryFilename[filename].push(modeledMethod);
// Add the new modeled methods, potentially overwriting existing modeled methods
// but not removing existing modeled methods that are not in the new set.
for (const externalApiUsage of externalApiUsages) {
const method = newModeledMethods[externalApiUsage.signature];
if (method) {
const filename = createFilenameForLibrary(externalApiUsage.library);
methodsByLibraryFilename[filename][method.signature] = method;
}
}
const result: Record<string, string> = {};
for (const [filename, methods] of Object.entries(methodsByLibraryFilename)) {
result[filename] = createDataExtensionYaml(language, methods);
result[filename] = createDataExtensionYaml(
language,
Object.values(methods),
);
}
return result;
@@ -127,7 +154,8 @@ export function createDataExtensionYamlsForFrameworkMode(
databaseName: string,
language: string,
externalApiUsages: ExternalApiUsage[],
modeledMethods: Record<string, ModeledMethod>,
newModeledMethods: Record<string, ModeledMethod>,
existingModeledMethods: Record<string, Record<string, ModeledMethod>>,
prefix = "models/",
suffix = ".model",
): Record<string, string> {
@@ -136,16 +164,28 @@ export function createDataExtensionYamlsForFrameworkMode(
.slice(1)
.map((part) => sanitizeExtensionPackName(part))
.join("-");
const filename = `${prefix}${libraryName}${suffix}.yml`;
const methods = externalApiUsages
.map((externalApiUsage) => modeledMethods[externalApiUsage.signature])
.filter((modeledMethod) => modeledMethod !== undefined);
const methods: Record<string, ModeledMethod> = {};
// First populate methodsByLibraryFilename with any existing modeled methods.
for (const [signature, method] of Object.entries(
existingModeledMethods[filename] || {},
)) {
methods[signature] = method;
}
// Add the new modeled methods, potentially overwriting existing modeled methods
// but not removing existing modeled methods that are not in the new set.
for (const externalApiUsage of externalApiUsages) {
const modeledMethod = newModeledMethods[externalApiUsage.signature];
if (modeledMethod) {
methods[modeledMethod.signature] = modeledMethod;
}
}
return {
[`${prefix}${libraryName}${suffix}.yml`]: createDataExtensionYaml(
language,
methods,
),
[filename]: createDataExtensionYaml(language, Object.values(methods)),
};
}

481
extensions/ql-vscode/test/unit-tests/data-extensions-editor/yaml.test.ts
View File

@@ -76,7 +76,7 @@ describe("createDataExtensionYaml", () => {
});
describe("createDataExtensionYamlsForApplicationMode", () => {
it("creates the correct YAML files", () => {
it("creates the correct YAML files when there are no existing modeled methods", () => {
const yaml = createDataExtensionYamlsForApplicationMode(
"java",
[
@@ -262,6 +262,7 @@ describe("createDataExtensionYamlsForApplicationMode", () => {
methodParameters: "(String,String,String)",
},
},
{},
);
expect(yaml).toEqual({
@@ -312,10 +313,290 @@ describe("createDataExtensionYamlsForApplicationMode", () => {
`,
});
});
it("creates the correct YAML files when there are existing modeled methods", () => {
const yaml = createDataExtensionYamlsForApplicationMode(
"java",
[
{
library: "sql2o",
libraryVersion: "1.6.0",
signature: "org.sql2o.Connection#createQuery(String)",
packageName: "org.sql2o",
typeName: "Connection",
methodName: "createQuery",
methodParameters: "(String)",
supported: true,
supportedType: "sink",
usages: [
{
label: "createQuery(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 15,
startColumn: 13,
endLine: 15,
endColumn: 56,
},
classification: CallClassification.Source,
},
{
label: "createQuery(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 26,
startColumn: 13,
endLine: 26,
endColumn: 39,
},
classification: CallClassification.Source,
},
],
},
{
library: "sql2o",
libraryVersion: "1.6.0",
signature: "org.sql2o.Query#executeScalar(Class)",
packageName: "org.sql2o",
typeName: "Query",
methodName: "executeScalar",
methodParameters: "(Class)",
supported: true,
supportedType: "neutral",
usages: [
{
label: "executeScalar(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 15,
startColumn: 13,
endLine: 15,
endColumn: 85,
},
classification: CallClassification.Source,
},
{
label: "executeScalar(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 26,
startColumn: 13,
endLine: 26,
endColumn: 68,
},
classification: CallClassification.Source,
},
],
},
{
library: "sql2o",
libraryVersion: "2.5.0-alpha1",
signature: "org.sql2o.Sql2o#Sql2o(String,String,String)",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "Sql2o",
methodParameters: "(String,String,String)",
supported: false,
supportedType: "none",
usages: [
{
label: "new Sql2o(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 10,
startColumn: 33,
endLine: 10,
endColumn: 88,
},
classification: CallClassification.Source,
},
],
},
{
library: "spring-boot",
libraryVersion: "3.0.2",
signature:
"org.springframework.boot.SpringApplication#run(Class,String[])",
packageName: "org.springframework.boot",
typeName: "SpringApplication",
methodName: "run",
methodParameters: "(Class,String[])",
supported: false,
supportedType: "none",
usages: [
{
label: "run(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/Sql2oExampleApplication.java",
startLine: 9,
startColumn: 9,
endLine: 9,
endColumn: 66,
},
classification: CallClassification.Source,
},
],
},
{
library: "rt",
signature: "java.io.PrintStream#println(String)",
packageName: "java.io",
typeName: "PrintStream",
methodName: "println",
methodParameters: "(String)",
supported: true,
supportedType: "summary",
usages: [
{
label: "println(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 29,
startColumn: 9,
endLine: 29,
endColumn: 49,
},
classification: CallClassification.Source,
},
],
},
],
{
"org.sql2o.Connection#createQuery(String)": {
type: "sink",
input: "Argument[0]",
output: "",
kind: "sql",
provenance: "df-generated",
signature: "org.sql2o.Connection#createQuery(String)",
packageName: "org.sql2o",
typeName: "Connection",
methodName: "createQuery",
methodParameters: "(String)",
},
"org.springframework.boot.SpringApplication#run(Class,String[])": {
type: "neutral",
input: "",
output: "",
kind: "summary",
provenance: "manual",
signature:
"org.springframework.boot.SpringApplication#run(Class,String[])",
packageName: "org.springframework.boot",
typeName: "SpringApplication",
methodName: "run",
methodParameters: "(Class,String[])",
},
"org.sql2o.Sql2o#Sql2o(String,String,String)": {
type: "sink",
input: "Argument[0]",
output: "",
kind: "jndi",
provenance: "manual",
signature: "org.sql2o.Sql2o#Sql2o(String,String,String)",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "Sql2o",
methodParameters: "(String,String,String)",
},
},
{
"models/sql2o.model.yml": {
"org.sql2o.Connection#createQuery(String)": {
type: "neutral",
input: "",
output: "",
kind: "summary",
provenance: "manual",
signature: "org.sql2o.Connection#createQuery(String)",
packageName: "org.sql2o",
typeName: "Connection",
methodName: "createQuery",
methodParameters: "(String)",
},
"org.sql2o.Query#executeScalar(Class)": {
type: "neutral",
input: "",
output: "",
kind: "summary",
provenance: "manual",
signature: "org.sql2o.Query#executeScalar(Class)",
packageName: "org.sql2o",
typeName: "Query",
methodName: "executeScalar",
methodParameters: "(Class)",
},
},
"models/gson.model.yml": {
"com.google.gson.TypeAdapter#fromJsonTree(JsonElement)": {
type: "summary",
input: "Argument[this]",
output: "ReturnValue",
kind: "taint",
provenance: "df-generated",
signature: "com.google.gson.TypeAdapter#fromJsonTree(JsonElement)",
packageName: "com.google.gson",
typeName: "TypeAdapter",
methodName: "fromJsonTree",
methodParameters: "(JsonElement)",
},
},
},
);
expect(yaml).toEqual({
"models/sql2o.model.yml": `extensions:
- addsTo:
pack: codeql/java-all
extensible: sourceModel
data: []
- addsTo:
pack: codeql/java-all
extensible: sinkModel
data:
- ["org.sql2o","Connection",true,"createQuery","(String)","","Argument[0]","sql","df-generated"]
- ["org.sql2o","Sql2o",true,"Sql2o","(String,String,String)","","Argument[0]","jndi","manual"]
- addsTo:
pack: codeql/java-all
extensible: summaryModel
data: []
- addsTo:
pack: codeql/java-all
extensible: neutralModel
data:
- ["org.sql2o","Query","executeScalar","(Class)","summary","manual"]
`,
"models/spring-boot.model.yml": `extensions:
- addsTo:
pack: codeql/java-all
extensible: sourceModel
data: []
- addsTo:
pack: codeql/java-all
extensible: sinkModel
data: []
- addsTo:
pack: codeql/java-all
extensible: summaryModel
data: []
- addsTo:
pack: codeql/java-all
extensible: neutralModel
data:
- ["org.springframework.boot","SpringApplication","run","(Class,String[])","summary","manual"]
`,
});
});
});
describe("createDataExtensionYamlsForFrameworkMode", () => {
it("creates the correct YAML files", () => {
it("creates the correct YAML files when there are no existing modeled methods", () => {
const yaml = createDataExtensionYamlsForFrameworkMode(
"github/sql2o",
"java",
@@ -438,6 +719,7 @@ describe("createDataExtensionYamlsForFrameworkMode", () => {
methodParameters: "(String,String,String)",
},
},
{},
);
expect(yaml).toEqual({
@@ -463,6 +745,201 @@ describe("createDataExtensionYamlsForFrameworkMode", () => {
pack: codeql/java-all
extensible: neutralModel
data: []
`,
});
});
it("creates the correct YAML files when there are existing modeled methods", () => {
const yaml = createDataExtensionYamlsForFrameworkMode(
"github/sql2o",
"java",
[
{
library: "sql2o",
signature: "org.sql2o.Connection#createQuery(String)",
packageName: "org.sql2o",
typeName: "Connection",
methodName: "createQuery",
methodParameters: "(String)",
supported: true,
supportedType: "sink",
usages: [
{
label: "createQuery(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 15,
startColumn: 13,
endLine: 15,
endColumn: 56,
},
classification: CallClassification.Source,
},
{
label: "createQuery(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 26,
startColumn: 13,
endLine: 26,
endColumn: 39,
},
classification: CallClassification.Source,
},
],
},
{
library: "sql2o",
signature: "org.sql2o.Query#executeScalar(Class)",
packageName: "org.sql2o",
typeName: "Query",
methodName: "executeScalar",
methodParameters: "(Class)",
supported: true,
supportedType: "neutral",
usages: [
{
label: "executeScalar(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 15,
startColumn: 13,
endLine: 15,
endColumn: 85,
},
classification: CallClassification.Source,
},
{
label: "executeScalar(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 26,
startColumn: 13,
endLine: 26,
endColumn: 68,
},
classification: CallClassification.Source,
},
],
},
{
library: "sql2o",
signature: "org.sql2o.Sql2o#Sql2o(String,String,String)",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "Sql2o",
methodParameters: "(String,String,String)",
supported: false,
supportedType: "none",
usages: [
{
label: "new Sql2o(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 10,
startColumn: 33,
endLine: 10,
endColumn: 88,
},
classification: CallClassification.Source,
},
],
},
],
{
"org.sql2o.Connection#createQuery(String)": {
type: "sink",
input: "Argument[0]",
output: "",
kind: "sql",
provenance: "df-generated",
signature: "org.sql2o.Connection#createQuery(String)",
packageName: "org.sql2o",
typeName: "Connection",
methodName: "createQuery",
methodParameters: "(String)",
},
"org.sql2o.Sql2o#Sql2o(String,String,String)": {
type: "sink",
input: "Argument[0]",
output: "",
kind: "jndi",
provenance: "manual",
signature: "org.sql2o.Sql2o#Sql2o(String,String,String)",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "Sql2o",
methodParameters: "(String,String,String)",
},
},
{
"models/sql2o.model.yml": {
"org.sql2o.Connection#createQuery(String)": {
type: "neutral",
input: "",
output: "",
kind: "summary",
provenance: "manual",
signature: "org.sql2o.Connection#createQuery(String)",
packageName: "org.sql2o",
typeName: "Connection",
methodName: "createQuery",
methodParameters: "(String)",
},
"org.sql2o.Query#executeScalar(Class)": {
type: "neutral",
input: "",
output: "",
kind: "summary",
provenance: "manual",
signature: "org.sql2o.Query#executeScalar(Class)",
packageName: "org.sql2o",
typeName: "Query",
methodName: "executeScalar",
methodParameters: "(Class)",
},
},
"models/gson.model.yml": {
"com.google.gson.TypeAdapter#fromJsonTree(JsonElement)": {
type: "summary",
input: "Argument[this]",
output: "ReturnValue",
kind: "taint",
provenance: "df-generated",
signature: "com.google.gson.TypeAdapter#fromJsonTree(JsonElement)",
packageName: "com.google.gson",
typeName: "TypeAdapter",
methodName: "fromJsonTree",
methodParameters: "(JsonElement)",
},
},
},
);
expect(yaml).toEqual({
"models/sql2o.model.yml": `extensions:
- addsTo:
pack: codeql/java-all
extensible: sourceModel
data: []
- addsTo:
pack: codeql/java-all
extensible: sinkModel
data:
- ["org.sql2o","Connection",true,"createQuery","(String)","","Argument[0]","sql","df-generated"]
- ["org.sql2o","Sql2o",true,"Sql2o","(String,String,String)","","Argument[0]","jndi","manual"]
- addsTo:
pack: codeql/java-all
extensible: summaryModel
data: []
- addsTo:
pack: codeql/java-all
extensible: neutralModel
data:
- ["org.sql2o","Query","executeScalar","(Class)","summary","manual"]
`,
});
});

6
extensions/ql-vscode/test/vscode-tests/cli-integration/data-extensions-editor/modeled-method-fs.test.ts
View File

@@ -142,8 +142,8 @@ describe("modeled-method-fs", () => {
const modelFiles = await listModelFiles(extensionPackPath, cli);
expect(modelFiles).toEqual(
new Set([
join(extensionPackPath, "models", "library1.model.yml"),
join(extensionPackPath, "models", "library2.model.yml"),
join("models", "library1.model.yml"),
join("models", "library2.model.yml"),
]),
);
});
@@ -160,7 +160,7 @@ describe("modeled-method-fs", () => {
const modelFiles = await listModelFiles(extensionPackPath, cli);
expect(modelFiles).toEqual(
new Set([join(extensionPackPath, "models", "library1.model.yml")]),
new Set([join("models", "library1.model.yml")]),
);
});
});