Merge branch 'main' into robertbrignull/data-modeled-methods-tests

This commit is contained in:
Robert
2023-07-17 11:06:40 +01:00
26 changed files with 869 additions and 294 deletions

View File

@@ -0,0 +1 @@
<svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg" fill="none"><path fill-rule="evenodd" clip-rule="evenodd" d="M4 2h8v4c.341.035.677.112 1 .23V1H3v8.48l1-1.75V2zm2.14 8L5 8 4 9.75 3.29 11 1 15h8l-2.29-4-.57-1zm-3.42 4l1.72-3L5 10l.56 1 1.72 3H2.72zm6.836-6.41a3.5 3.5 0 1 1 3.888 5.82 3.5 3.5 0 0 1-3.888-5.82zm.555 4.989a2.5 2.5 0 1 0 2.778-4.157 2.5 2.5 0 0 0-2.778 4.157z" fill="#C5C5C5"/></svg>

After

Width:  |  Height:  |  Size: 431 B

View File

@@ -0,0 +1 @@
<svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg" fill="none"><path fill-rule="evenodd" clip-rule="evenodd" d="M4 2h8v4c.341.035.677.112 1 .23V1H3v8.48l1-1.75V2zm2.14 8L5 8 4 9.75 3.29 11 1 15h8l-2.29-4-.57-1zm-3.42 4l1.72-3L5 10l.56 1 1.72 3H2.72zm6.836-6.41a3.5 3.5 0 1 1 3.888 5.82 3.5 3.5 0 0 1-3.888-5.82zm.555 4.989a2.5 2.5 0 1 0 2.778-4.157 2.5 2.5 0 0 0-2.778 4.157z" fill="#424242"/></svg>

After

Width:  |  Height:  |  Size: 431 B

View File

@@ -19,6 +19,7 @@ export type WebviewPanelConfig = {
viewColumn: ViewColumn;
view: WebviewView;
preserveFocus?: boolean;
iconPath?: Uri | { dark: Uri; light: Uri };
additionalOptions?: WebviewPanelOptions & WebviewOptions;
allowWasmEval?: boolean;
};
@@ -86,6 +87,8 @@ export abstract class AbstractWebview<
);
this.panel = panel;
this.panel.iconPath = config.iconPath;
this.setupPanel(panel, config);
this.panelResolves.forEach((resolve) => resolve(panel));

View File

@@ -140,6 +140,12 @@ export function parsePredictedClassifications(
input: "",
output: "",
provenance: "ai-generated",
signature,
// predictedBySignature[signature] always has at least element
packageName: predictedMethods[0].package,
typeName: predictedMethods[0].type,
methodName: predictedMethods[0].name,
methodParameters: predictedMethods[0].signature,
};
continue;
}
@@ -157,6 +163,11 @@ export function parsePredictedClassifications(
input: sink.input ?? "",
output: sink.output ?? "",
provenance: "ai-generated",
signature,
packageName: sink.package,
typeName: sink.type,
methodName: sink.name,
methodParameters: sink.signature,
};
}

View File

@@ -1,5 +1,11 @@
import { DecodedBqrsChunk } from "../common/bqrs-cli-types";
import { Call, ExternalApiUsage } from "./external-api-usage";
import {
Call,
CallClassification,
ExternalApiUsage,
} from "./external-api-usage";
import { ModeledMethodType } from "./modeled-method";
import { parseLibraryFilename } from "./library";
export function decodeBqrsToExternalApiUsages(
chunk: DecodedBqrsChunk,
@@ -10,7 +16,10 @@ export function decodeBqrsToExternalApiUsages(
const usage = tuple[0] as Call;
const signature = tuple[1] as string;
const supported = (tuple[2] as string) === "true";
const library = tuple[4] as string;
let library = tuple[4] as string;
let libraryVersion: string | undefined = tuple[5] as string;
const type = tuple[6] as ModeledMethodType;
const classification = tuple[8] as CallClassification;
const [packageWithType, methodDeclaration] = signature.split("#");
@@ -30,21 +39,41 @@ export function decodeBqrsToExternalApiUsages(
methodDeclaration.indexOf("("),
);
// For Java, we'll always get back a .jar file, and the library version may be bad because not all library authors
// properly specify the version. Therefore, we'll always try to parse the name and version from the library filename
// for Java.
if (library.endsWith(".jar") || libraryVersion === "") {
const { name, version } = parseLibraryFilename(library);
library = name;
if (version) {
libraryVersion = version;
}
}
if (libraryVersion === "") {
libraryVersion = undefined;
}
if (!methodsByApiName.has(signature)) {
methodsByApiName.set(signature, {
library,
libraryVersion,
signature,
packageName,
typeName,
methodName,
methodParameters,
supported,
supportedType: type,
usages: [],
});
}
const method = methodsByApiName.get(signature)!;
method.usages.push(usage);
method.usages.push({
...usage,
classification,
});
});
return Array.from(methodsByApiName.values());

View File

@@ -76,6 +76,14 @@ export class DataExtensionsEditorView extends AbstractWebview<
viewColumn: ViewColumn.Active,
preserveFocus: true,
view: "data-extensions-editor",
iconPath: {
dark: Uri.file(
join(this.ctx.extensionPath, "media/dark/symbol-misc.svg"),
),
light: Uri.file(
join(this.ctx.extensionPath, "media/light/symbol-misc.svg"),
),
},
};
}
@@ -311,11 +319,11 @@ export class DataExtensionsEditorView extends AbstractWebview<
queryRunner: this.queryRunner,
queryStorageDir: this.queryStorageDir,
databaseItem: addedDatabase ?? this.databaseItem,
onResults: async (results) => {
onResults: async (modeledMethods) => {
const modeledMethodsByName: Record<string, ModeledMethod> = {};
for (const result of results) {
modeledMethodsByName[result.signature] = result.modeledMethod;
for (const modeledMethod of modeledMethods) {
modeledMethodsByName[modeledMethod.signature] = modeledMethod;
}
await this.postMessage({

View File

@@ -1,15 +1,27 @@
import { ResolvableLocationValue } from "../common/bqrs-cli-types";
import { ModeledMethodType } from "./modeled-method";
export type Call = {
label: string;
url: ResolvableLocationValue;
};
export type ExternalApiUsage = {
export enum CallClassification {
Unknown = "unknown",
Source = "source",
Test = "test",
Generated = "generated",
}
export type Usage = Call & {
classification: CallClassification;
};
export interface MethodSignature {
/**
* Contains the name of the library containing the method declaration, e.g. `sql2o-1.6.0.jar` or `System.Runtime.dll`
* Contains the version of the library if it can be determined by CodeQL, e.g. `4.2.2.2`
*/
library: string;
libraryVersion?: string;
/**
* A unique signature that can be used to identify this external API usage.
*
@@ -25,10 +37,18 @@ export type ExternalApiUsage = {
* The method parameters, including enclosing parentheses, e.g. `(String, String)`
*/
methodParameters: string;
}
export interface ExternalApiUsage extends MethodSignature {
/**
* Contains the name of the library containing the method declaration, e.g. `sql2o-1.6.0.jar` or `System.Runtime.dll`
*/
library: string;
/**
* Is this method already supported by CodeQL standard libraries.
* If so, there is no need for the user to model it themselves.
*/
supported: boolean;
usages: Call[];
};
supportedType: ModeledMethodType;
usages: Usage[];
}

View File

@@ -8,10 +8,7 @@ import { extLogger } from "../common/logging/vscode";
import { extensiblePredicateDefinitions } from "./predicates";
import { ProgressCallback } from "../common/vscode/progress";
import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
import {
ModeledMethodType,
ModeledMethodWithSignature,
} from "./modeled-method";
import { ModeledMethod, ModeledMethodType } from "./modeled-method";
import { redactableError } from "../common/errors";
import { QueryResultType } from "../query-server/new-messages";
import { file } from "tmp-promise";
@@ -27,7 +24,7 @@ type FlowModelOptions = {
databaseItem: DatabaseItem;
progress: ProgressCallback;
token: CancellationToken;
onResults: (results: ModeledMethodWithSignature[]) => void | Promise<void>;
onResults: (results: ModeledMethod[]) => void | Promise<void>;
};
async function resolveQueries(
@@ -79,7 +76,7 @@ async function getModeledMethodsFromFlow(
progress,
token,
}: Omit<FlowModelOptions, "onResults">,
): Promise<ModeledMethodWithSignature[]> {
): Promise<ModeledMethod[]> {
if (queryPath === undefined) {
void showAndLogExceptionWithTelemetry(
extLogger,

View File

@@ -0,0 +1,58 @@
import { basename, extname } from "../common/path";
// From the semver package using
// const { re, t } = require("semver/internal/re");
// console.log(re[t.LOOSE]);
// Modifications:
// - Added version named group which does not capture the v prefix
// - Removed the ^ and $ anchors
// - Made the minor and patch versions optional
// - Added a hyphen to the start of the version
// - Added a dot as a valid separator between the version and the label
// - Made the patch version optional even if a label is given
// This will match any semver string at the end of a larger string
const semverRegex =
/-[v=\s]*(?<version>([0-9]+)(\.([0-9]+)(?:(\.([0-9]+))?(?:[-.]?((?:[0-9]+|\d*[a-zA-Z-][a-zA-Z0-9-]*)(?:\.(?:[0-9]+|\d*[a-zA-Z-][a-zA-Z0-9-]*))*))?(?:\+([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?)?)?)/g;
export interface Library {
name: string;
version?: string;
}
export function parseLibraryFilename(filename: string): Library {
let libraryName = basename(filename);
const extension = extname(libraryName);
libraryName = libraryName.slice(0, -extension.length);
let libraryVersion: string | undefined;
let match: RegExpMatchArray | null = null;
// Reset the regex
semverRegex.lastIndex = 0;
// Find the last occurence of the regex within the library name
// eslint-disable-next-line no-constant-condition
while (true) {
const currentMatch = semverRegex.exec(libraryName);
if (currentMatch === null) {
break;
}
match = currentMatch;
}
if (match?.groups) {
libraryVersion = match.groups?.version;
// Remove everything after the start of the match
libraryName = libraryName.slice(0, match.index);
}
// Remove any leading or trailing hyphens or dots
libraryName = libraryName.replaceAll(/^[.-]+|[.-]+$/g, "");
return {
name: libraryName,
version: libraryVersion,
};
}

View File

@@ -1,3 +1,5 @@
import { MethodSignature } from "./external-api-usage";
export type ModeledMethodType =
| "none"
| "source"
@@ -17,15 +19,10 @@ export type Provenance =
// Entered by the user in the editor manually
| "manual";
export type ModeledMethod = {
export interface ModeledMethod extends MethodSignature {
type: ModeledMethodType;
input: string;
output: string;
kind: string;
provenance: Provenance;
};
export type ModeledMethodWithSignature = {
signature: string;
modeledMethod: ModeledMethod;
};
}

View File

@@ -1,20 +1,9 @@
import { ExternalApiUsage } from "./external-api-usage";
import {
ModeledMethod,
ModeledMethodType,
ModeledMethodWithSignature,
Provenance,
} from "./modeled-method";
export type ExternalApiUsageByType = {
externalApiUsage: ExternalApiUsage;
modeledMethod: ModeledMethod;
};
import { ModeledMethod, ModeledMethodType, Provenance } from "./modeled-method";
export type ExtensiblePredicateDefinition = {
extensiblePredicate: string;
generateMethodDefinition: (method: ExternalApiUsageByType) => Tuple[];
readModeledMethod: (row: Tuple[]) => ModeledMethodWithSignature;
generateMethodDefinition: (method: ModeledMethod) => Tuple[];
readModeledMethod: (row: Tuple[]) => ModeledMethod;
supportedKinds?: string[];
};
@@ -36,25 +25,27 @@ export const extensiblePredicateDefinitions: Record<
// string output, string kind, string provenance
// );
generateMethodDefinition: (method) => [
method.externalApiUsage.packageName,
method.externalApiUsage.typeName,
method.packageName,
method.typeName,
true,
method.externalApiUsage.methodName,
method.externalApiUsage.methodParameters,
method.methodName,
method.methodParameters,
"",
method.modeledMethod.output,
method.modeledMethod.kind,
method.modeledMethod.provenance,
method.output,
method.kind,
method.provenance,
],
readModeledMethod: (row) => ({
type: "source",
input: "",
output: row[6] as string,
kind: row[7] as string,
provenance: row[8] as Provenance,
signature: readRowToMethod(row),
modeledMethod: {
type: "source",
input: "",
output: row[6] as string,
kind: row[7] as string,
provenance: row[8] as Provenance,
},
packageName: row[0] as string,
typeName: row[1] as string,
methodName: row[3] as string,
methodParameters: row[4] as string,
}),
supportedKinds: ["remote"],
},
@@ -65,25 +56,27 @@ export const extensiblePredicateDefinitions: Record<
// string input, string kind, string provenance
// );
generateMethodDefinition: (method) => [
method.externalApiUsage.packageName,
method.externalApiUsage.typeName,
method.packageName,
method.typeName,
true,
method.externalApiUsage.methodName,
method.externalApiUsage.methodParameters,
method.methodName,
method.methodParameters,
"",
method.modeledMethod.input,
method.modeledMethod.kind,
method.modeledMethod.provenance,
method.input,
method.kind,
method.provenance,
],
readModeledMethod: (row) => ({
type: "sink",
input: row[6] as string,
output: "",
kind: row[7] as string,
provenance: row[8] as Provenance,
signature: readRowToMethod(row),
modeledMethod: {
type: "sink",
input: row[6] as string,
output: "",
kind: row[7] as string,
provenance: row[8] as Provenance,
},
packageName: row[0] as string,
typeName: row[1] as string,
methodName: row[3] as string,
methodParameters: row[4] as string,
}),
supportedKinds: ["sql", "xss", "logging"],
},
@@ -94,26 +87,28 @@ export const extensiblePredicateDefinitions: Record<
// string input, string output, string kind, string provenance
// );
generateMethodDefinition: (method) => [
method.externalApiUsage.packageName,
method.externalApiUsage.typeName,
method.packageName,
method.typeName,
true,
method.externalApiUsage.methodName,
method.externalApiUsage.methodParameters,
method.methodName,
method.methodParameters,
"",
method.modeledMethod.input,
method.modeledMethod.output,
method.modeledMethod.kind,
method.modeledMethod.provenance,
method.input,
method.output,
method.kind,
method.provenance,
],
readModeledMethod: (row) => ({
type: "summary",
input: row[6] as string,
output: row[7] as string,
kind: row[8] as string,
provenance: row[9] as Provenance,
signature: readRowToMethod(row),
modeledMethod: {
type: "summary",
input: row[6] as string,
output: row[7] as string,
kind: row[8] as string,
provenance: row[9] as Provenance,
},
packageName: row[0] as string,
typeName: row[1] as string,
methodName: row[3] as string,
methodParameters: row[4] as string,
}),
supportedKinds: ["taint", "value"],
},
@@ -123,22 +118,24 @@ export const extensiblePredicateDefinitions: Record<
// string package, string type, string name, string signature, string kind, string provenance
// );
generateMethodDefinition: (method) => [
method.externalApiUsage.packageName,
method.externalApiUsage.typeName,
method.externalApiUsage.methodName,
method.externalApiUsage.methodParameters,
method.modeledMethod.kind,
method.modeledMethod.provenance,
method.packageName,
method.typeName,
method.methodName,
method.methodParameters,
method.kind,
method.provenance,
],
readModeledMethod: (row) => ({
type: "neutral",
input: "",
output: "",
kind: row[4] as string,
provenance: row[5] as Provenance,
signature: `${row[0]}.${row[1]}#${row[2]}${row[3]}`,
modeledMethod: {
type: "neutral",
input: "",
output: "",
kind: row[4] as string,
provenance: row[5] as Provenance,
},
packageName: row[0] as string,
typeName: row[1] as string,
methodName: row[2] as string,
methodParameters: row[3] as string,
}),
supportedKinds: ["summary", "source", "sink"],
},

View File

@@ -22,12 +22,16 @@ class ExternalApi extends CallableMethod {
private Call aUsage(ExternalApi api) { result.getTarget().getUnboundDeclaration() = api }
from ExternalApi api, string apiName, boolean supported, Call usage
from
ExternalApi api, string apiName, boolean supported, Call usage, string type, string classification
where
apiName = api.getApiName() and
supported = isSupported(api) and
usage = aUsage(api)
select usage, apiName, supported.toString(), "supported", api.getFile().getBaseName(), "library"
usage = aUsage(api) and
type = supportedType(api) and
classification = methodClassification(usage)
select usage, apiName, supported.toString(), "supported", api.dllName(), api.dllVersion(), type,
"type", classification, "classification"
`,
frameworkModeQuery: `/**
* @name Public methods
@@ -46,12 +50,13 @@ class PublicMethod extends CallableMethod {
PublicMethod() { this.fromSource() and not this.getFile() instanceof TestFile }
}
from PublicMethod publicMethod, string apiName, boolean supported
from PublicMethod publicMethod, string apiName, boolean supported, string type
where
apiName = publicMethod.getApiName() and
supported = isSupported(publicMethod)
supported = isSupported(publicMethod) and
type = supportedType(publicMethod)
select publicMethod, apiName, supported.toString(), "supported",
publicMethod.getFile().getBaseName(), "library"
publicMethod.getFile().getBaseName(), "library", type, "type", "unknown", "classification"
`,
dependencies: {
"AutomodelVsCode.qll": `/** Provides classes and predicates related to handling APIs for the VS Code extension. */
@@ -66,6 +71,7 @@ private import semmle.code.csharp.dataflow.internal.DataFlowPrivate
private import semmle.code.csharp.dataflow.internal.DataFlowDispatch as DataFlowDispatch
private import semmle.code.csharp.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
private import semmle.code.csharp.dataflow.internal.TaintTrackingPrivate
private import semmle.code.csharp.frameworks.Test
private import semmle.code.csharp.security.dataflow.flowsources.Remote
pragma[nomagic]
@@ -105,7 +111,7 @@ class CallableMethod extends DotNet::Declaration {
bindingset[this]
private string getSignature() {
result =
nestedName(this.getDeclaringType().getUnboundDeclaration()) + "." + this.getName() + "(" +
nestedName(this.getDeclaringType().getUnboundDeclaration()) + "#" + this.getName() + "(" +
parameterQualifiedTypeNamesToString(this) + ")"
}
@@ -119,7 +125,23 @@ class CallableMethod extends DotNet::Declaration {
* Gets the namespace and signature of this API.
*/
bindingset[this]
string getApiName() { result = this.getNamespace() + "#" + this.getSignature() }
string getApiName() { result = this.getNamespace() + "." + this.getSignature() }
private string getDllName() { result = this.getLocation().(Assembly).getName() }
private string getDllVersion() { result = this.getLocation().(Assembly).getVersion().toString() }
string dllName() {
result = this.getDllName()
or
not exists(this.getDllName()) and result = this.getFile().getBaseName()
}
string dllVersion() {
result = this.getDllVersion()
or
not exists(this.getDllVersion()) and result = ""
}
/** Gets a node that is an input to a call to this API. */
private ArgumentNode getAnInput() {
@@ -180,6 +202,25 @@ boolean isSupported(CallableMethod callableMethod) {
result = false
}
string supportedType(CallableMethod method) {
method.isSink() and result = "sink"
or
method.isSource() and result = "source"
or
method.hasSummary() and result = "summary"
or
method.isNeutral() and result = "neutral"
or
not method.isSupported() and result = ""
}
string methodClassification(Call method) {
method.getFile() instanceof TestFile and result = "test"
or
not method.getFile() instanceof TestFile and
result = "source"
}
/**
* Gets the nested name of the declaration.
*

View File

@@ -16,17 +16,19 @@ class ExternalApi extends CallableMethod {
ExternalApi() { not this.fromSource() }
}
private Call aUsage(ExternalApi api) {
result.getCallee().getSourceDeclaration() = api and
not result.getFile() instanceof GeneratedFile
}
private Call aUsage(ExternalApi api) { result.getCallee().getSourceDeclaration() = api }
from ExternalApi externalApi, string apiName, boolean supported, Call usage
from
ExternalApi externalApi, string apiName, boolean supported, Call usage, string type,
string classification
where
apiName = externalApi.getApiName() and
supported = isSupported(externalApi) and
usage = aUsage(externalApi)
select usage, apiName, supported.toString(), "supported", externalApi.jarContainer(), "library"
usage = aUsage(externalApi) and
type = supportedType(externalApi) and
classification = methodClassification(usage)
select usage, apiName, supported.toString(), "supported", externalApi.jarContainer(),
externalApi.jarVersion(), type, "type", classification, "classification"
`,
frameworkModeQuery: `/**
* @name Public methods
@@ -41,12 +43,14 @@ import AutomodelVsCode
class PublicMethodFromSource extends CallableMethod, ModelApi { }
from PublicMethodFromSource publicMethod, string apiName, boolean supported
from PublicMethodFromSource publicMethod, string apiName, boolean supported, string type
where
apiName = publicMethod.getApiName() and
supported = isSupported(publicMethod)
supported = isSupported(publicMethod) and
type = supportedType(publicMethod)
select publicMethod, apiName, supported.toString(), "supported",
publicMethod.getCompilationUnit().getParentContainer().getBaseName(), "library"
publicMethod.getCompilationUnit().getParentContainer().getBaseName(), "library", type, "type",
"unknown", "classification"
`,
dependencies: {
"AutomodelVsCode.qll": `/** Provides classes and predicates related to handling APIs for the VS Code extension. */
@@ -71,7 +75,7 @@ private predicate isUninteresting(Callable c) {
/**
* A callable method from either the Standard Library, a 3rd party library or from the source.
*/
class CallableMethod extends Method {
class CallableMethod extends Callable {
CallableMethod() { not isUninteresting(this) }
/**
@@ -87,6 +91,10 @@ class CallableMethod extends Method {
result = this.getCompilationUnit().getParentContainer*().(JarFile).getBaseName()
}
private string getJarVersion() {
result = this.getCompilationUnit().getParentContainer*().(JarFile).getSpecificationVersion()
}
/**
* Gets the jar file containing this API. Normalizes the Java Runtime to "rt.jar" despite the presence of modules.
*/
@@ -96,6 +104,15 @@ class CallableMethod extends Method {
not exists(this.getJarName()) and result = "rt.jar"
}
/**
* Gets the version of the JAR file containing this API. Empty if no version is found in the JAR.
*/
string jarVersion() {
result = this.getJarVersion()
or
not exists(this.getJarVersion()) and result = ""
}
/** Gets a node that is an input to a call to this API. */
private DataFlow::Node getAnInput() {
exists(Call call | call.getCallee().getSourceDeclaration() = this |
@@ -147,6 +164,28 @@ boolean isSupported(CallableMethod method) {
not method.isSupported() and result = false
}
string supportedType(CallableMethod method) {
method.isSink() and result = "sink"
or
method.isSource() and result = "source"
or
method.hasSummary() and result = "summary"
or
method.isNeutral() and result = "neutral"
or
not method.isSupported() and result = ""
}
string methodClassification(Call method) {
isInTestFile(method.getLocation().getFile()) and result = "test"
or
method.getFile() instanceof GeneratedFile and result = "generated"
or
not isInTestFile(method.getLocation().getFile()) and
not method.getFile() instanceof GeneratedFile and
result = "source"
}
// The below is a copy of https://github.com/github/codeql/blob/249f9f863db1e94e3c46ca85b49fb0ec32f8ca92/java/ql/lib/semmle/code/java/dataflow/internal/ModelExclusions.qll
// to avoid the use of internal modules.
/** Holds if the given package \`p\` is a test package. */

View File

@@ -8,7 +8,11 @@ export type Query = {
* - supported: whether the external API is modeled. This should be a string representation of a boolean to satify the result pattern for a problem query.
* - "supported": a string literal. This is required to make the query a valid problem query.
* - libraryName: the name of the library that contains the external API. This is a string and usually the basename of a file.
* - "library": a string literal. This is required to make the query a valid problem query.
* - libraryVersion: the version of the library that contains the external API. This is a string and can be empty if the version cannot be determined.
* - type: the modeled kind of the method, either "sink", "source", "summary", or "neutral"
* - "type": a string literal. This is required to make the query a valid problem query.
* - classification: the classification of the use of the method, either "source", "test", "generated", or "unknown"
* - "classification: a string literal. This is required to make the query a valid problem query.
*/
applicationModeQuery: string;
/**
@@ -21,7 +25,11 @@ export type Query = {
* - supported: whether this method is modeled. This should be a string representation of a boolean to satify the result pattern for a problem query.
* - "supported": a string literal. This is required to make the query a valid problem query.
* - libraryName: an arbitrary string. This is required to make it match the structure of the application query.
* - "library": a string literal. This is required to make the query a valid problem query.
* - libraryVersion: an arbitrary string. This is required to make it match the structure of the application query.
* - type: the modeled kind of the method, either "sink", "source", "summary", or "neutral"
* - "type": a string literal. This is required to make the query a valid problem query.
* - "unknown": a string literal. This is required to make it match the structure of the application query.
* - "classification: a string literal. This is required to make the query a valid problem query.
*/
frameworkModeQuery: string;
dependencies?: {

View File

@@ -1,12 +1,10 @@
import Ajv from "ajv";
import { basename, extname } from "../common/path";
import { ExternalApiUsage } from "./external-api-usage";
import { ModeledMethod, ModeledMethodType } from "./modeled-method";
import {
ExtensiblePredicateDefinition,
extensiblePredicateDefinitions,
ExternalApiUsageByType,
} from "./predicates";
import * as dataSchemaJson from "./data-schema.json";
@@ -17,25 +15,15 @@ import { assertNever } from "../common/helpers-pure";
const ajv = new Ajv({ allErrors: true });
const dataSchemaValidate = ajv.compile(dataSchemaJson);
type ModeledExternalApiUsage = {
externalApiUsage: ExternalApiUsage;
modeledMethod?: ModeledMethod;
};
function createDataProperty(
methods: ModeledExternalApiUsage[],
methods: ModeledMethod[],
definition: ExtensiblePredicateDefinition,
) {
if (methods.length === 0) {
return " []";
}
const modeledMethods = methods.filter(
(method): method is ExternalApiUsageByType =>
method.modeledMethod !== undefined,
);
return `\n${modeledMethods
return `\n${methods
.map(
(method) =>
` - ${JSON.stringify(
@@ -47,11 +35,11 @@ function createDataProperty(
export function createDataExtensionYaml(
language: string,
modeledUsages: ModeledExternalApiUsage[],
modeledMethods: ModeledMethod[],
) {
const methodsByType: Record<
Exclude<ModeledMethodType, "none">,
ModeledExternalApiUsage[]
ModeledMethod[]
> = {
source: [],
sink: [],
@@ -59,11 +47,9 @@ export function createDataExtensionYaml(
neutral: [],
};
for (const modeledUsage of modeledUsages) {
const { modeledMethod } = modeledUsage;
for (const modeledMethod of modeledMethods) {
if (modeledMethod?.type && modeledMethod.type !== "none") {
methodsByType[modeledMethod.type].push(modeledUsage);
methodsByType[modeledMethod.type].push(modeledMethod);
}
}
@@ -113,32 +99,24 @@ export function createDataExtensionYamlsForApplicationMode(
externalApiUsages: ExternalApiUsage[],
modeledMethods: Record<string, ModeledMethod>,
): Record<string, string> {
const methodsByLibraryFilename: Record<string, ModeledExternalApiUsage[]> =
{};
const methodsByLibraryFilename: Record<string, ModeledMethod[]> = {};
for (const externalApiUsage of externalApiUsages) {
const modeledMethod = modeledMethods[externalApiUsage.signature];
if (!modeledMethod) {
continue;
}
const filename = createFilenameForLibrary(externalApiUsage.library);
methodsByLibraryFilename[filename] =
methodsByLibraryFilename[filename] || [];
methodsByLibraryFilename[filename].push({
externalApiUsage,
modeledMethod,
});
methodsByLibraryFilename[filename].push(modeledMethod);
}
const result: Record<string, string> = {};
for (const [filename, methods] of Object.entries(methodsByLibraryFilename)) {
const hasModeledMethods = methods.some(
(method) => method.modeledMethod !== undefined,
);
if (!hasModeledMethods) {
continue;
}
result[filename] = createDataExtensionYaml(language, methods);
}
@@ -159,10 +137,9 @@ export function createDataExtensionYamlsForFrameworkMode(
.map((part) => sanitizeExtensionPackName(part))
.join("-");
const methods = externalApiUsages.map((externalApiUsage) => ({
externalApiUsage,
modeledMethod: modeledMethods[externalApiUsage.signature],
}));
const methods = externalApiUsages
.map((externalApiUsage) => modeledMethods[externalApiUsage.signature])
.filter((modeledMethod) => modeledMethod !== undefined);
return {
[`${prefix}${libraryName}${suffix}.yml`]: createDataExtensionYaml(
@@ -172,29 +149,12 @@ export function createDataExtensionYamlsForFrameworkMode(
};
}
// From the semver package using
// const { re, t } = require("semver/internal/re");
// console.log(re[t.LOOSE]);
// Modified to remove the ^ and $ anchors
// This will match any semver string at the end of a larger string
const semverRegex =
/[v=\s]*([0-9]+)\.([0-9]+)\.([0-9]+)(?:-?((?:[0-9]+|\d*[a-zA-Z-][a-zA-Z0-9-]*)(?:\.(?:[0-9]+|\d*[a-zA-Z-][a-zA-Z0-9-]*))*))?(?:\+([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?/;
export function createFilenameForLibrary(
library: string,
prefix = "models/",
suffix = ".model",
) {
let libraryName = basename(library);
const extension = extname(libraryName);
libraryName = libraryName.slice(0, -extension.length);
const match = semverRegex.exec(libraryName);
if (match !== null) {
// Remove everything after the start of the match
libraryName = libraryName.slice(0, match.index);
}
let libraryName = library;
// Lowercase everything
libraryName = libraryName.toLowerCase();
@@ -249,14 +209,11 @@ export function loadDataExtensionYaml(
}
for (const row of data) {
const result = definition.readModeledMethod(row);
if (!result) {
const modeledMethod = definition.readModeledMethod(row);
if (!modeledMethod) {
continue;
}
const { signature, modeledMethod } = result;
modeledMethods[signature] = modeledMethod;
modeledMethods[modeledMethod.signature] = modeledMethod;
}
}

View File

@@ -4,6 +4,7 @@ import { ComponentMeta, ComponentStory } from "@storybook/react";
import { Mode } from "../../data-extensions-editor/shared/mode";
import { DataExtensionsEditor as DataExtensionsEditorComponent } from "../../view/data-extensions-editor/DataExtensionsEditor";
import { CallClassification } from "../../data-extensions-editor/external-api-usage";
export default {
title: "Data Extensions Editor/Data Extensions Editor",
@@ -32,13 +33,15 @@ DataExtensionsEditor.args = {
},
initialExternalApiUsages: [
{
library: "sql2o-1.6.0.jar",
library: "sql2o",
libraryVersion: "1.6.0",
signature: "org.sql2o.Connection#createQuery(String)",
packageName: "org.sql2o",
typeName: "Connection",
methodName: "createQuery",
methodParameters: "(String)",
supported: true,
supportedType: "summary",
usages: Array(10).fill({
label: "createQuery(...)",
url: {
@@ -48,16 +51,19 @@ DataExtensionsEditor.args = {
endLine: 15,
endColumn: 56,
},
classification: CallClassification.Source,
}),
},
{
library: "sql2o-1.6.0.jar",
library: "sql2o",
libraryVersion: "1.6.0",
signature: "org.sql2o.Query#executeScalar(Class)",
packageName: "org.sql2o",
typeName: "Query",
methodName: "executeScalar",
methodParameters: "(Class)",
supported: true,
supportedType: "neutral",
usages: Array(2).fill({
label: "executeScalar(...)",
url: {
@@ -67,16 +73,19 @@ DataExtensionsEditor.args = {
endLine: 15,
endColumn: 85,
},
classification: CallClassification.Source,
}),
},
{
library: "sql2o-1.6.0.jar",
library: "sql2o",
libraryVersion: "1.6.0",
signature: "org.sql2o.Sql2o#open()",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "open",
methodParameters: "()",
supported: false,
supportedType: "none",
usages: Array(28).fill({
label: "open(...)",
url: {
@@ -86,16 +95,18 @@ DataExtensionsEditor.args = {
endLine: 14,
endColumn: 35,
},
classification: CallClassification.Source,
}),
},
{
library: "rt.jar",
library: "rt",
signature: "java.io.PrintStream#println(String)",
packageName: "java.io",
typeName: "PrintStream",
methodName: "println",
methodParameters: "(String)",
supported: true,
supportedType: "summary",
usages: [
{
label: "println(...)",
@@ -106,11 +117,24 @@ DataExtensionsEditor.args = {
endLine: 29,
endColumn: 49,
},
classification: CallClassification.Source,
},
{
label: "println(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/test/java/org/example/HelloControllerTest.java",
startLine: 29,
startColumn: 9,
endLine: 29,
endColumn: 49,
},
classification: CallClassification.Test,
},
],
},
{
library: "spring-boot-3.0.2.jar",
library: "spring-boot",
libraryVersion: "3.0.2",
signature:
"org.springframework.boot.SpringApplication#run(Class,String[])",
packageName: "org.springframework.boot",
@@ -118,6 +142,7 @@ DataExtensionsEditor.args = {
methodName: "run",
methodParameters: "(Class,String[])",
supported: false,
supportedType: "none",
usages: Array(7).fill({
label: "run(...)",
url: {
@@ -127,16 +152,19 @@ DataExtensionsEditor.args = {
endLine: 9,
endColumn: 66,
},
classification: CallClassification.Source,
}),
},
{
library: "sql2o-1.6.0.jar",
library: "sql2o",
libraryVersion: "1.6.0",
signature: "org.sql2o.Sql2o#Sql2o(String,String,String)",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "Sql2o",
methodParameters: "(String,String,String)",
supported: false,
supportedType: "none",
usages: Array(106).fill({
label: "new Sql2o(...)",
url: {
@@ -145,27 +173,44 @@ DataExtensionsEditor.args = {
startColumn: 33,
endLine: 10,
endColumn: 88,
classification: CallClassification.Test,
},
}),
},
{
library: "sql2o-1.6.0.jar",
library: "sql2o",
libraryVersion: "1.6.0",
signature: "org.sql2o.Sql2o#Sql2o(String)",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "Sql2o",
methodParameters: "(String)",
supported: false,
usages: Array(4).fill({
label: "new Sql2o(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 23,
startColumn: 23,
endLine: 23,
endColumn: 36,
supportedType: "none",
usages: [
...Array(4).fill({
label: "new Sql2o(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 23,
startColumn: 23,
endLine: 23,
endColumn: 36,
},
classification: CallClassification.Test,
}),
{
label: "new Sql2o(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/build/generated/java/org/example/HelloControllerGenerated.java",
startLine: 23,
startColumn: 23,
endLine: 23,
endColumn: 36,
},
classification: CallClassification.Generated,
},
}),
],
},
],
initialModeledMethods: {
@@ -175,6 +220,11 @@ DataExtensionsEditor.args = {
output: "",
kind: "jndi-injection",
provenance: "df-generated",
signature: "org.sql2o.Sql2o#Sql2o(String)",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "Sql2o",
methodParameters: "(String)",
},
"org.sql2o.Connection#createQuery(String)": {
type: "summary",
@@ -182,6 +232,11 @@ DataExtensionsEditor.args = {
output: "ReturnValue",
kind: "taint",
provenance: "df-manual",
signature: "org.sql2o.Connection#createQuery(String)",
packageName: "org.sql2o",
typeName: "Connection",
methodName: "createQuery",
methodParameters: "(String)",
},
"org.sql2o.Sql2o#open()": {
type: "summary",
@@ -189,6 +244,11 @@ DataExtensionsEditor.args = {
output: "ReturnValue",
kind: "taint",
provenance: "manual",
signature: "org.sql2o.Sql2o#open()",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "open",
methodParameters: "()",
},
"org.sql2o.Query#executeScalar(Class)": {
type: "neutral",
@@ -196,6 +256,11 @@ DataExtensionsEditor.args = {
output: "",
kind: "",
provenance: "df-generated",
signature: "org.sql2o.Query#executeScalar(Class)",
packageName: "org.sql2o",
typeName: "Query",
methodName: "executeScalar",
methodParameters: "(Class)",
},
"org.sql2o.Sql2o#Sql2o(String,String,String)": {
type: "neutral",
@@ -203,6 +268,11 @@ DataExtensionsEditor.args = {
output: "",
kind: "",
provenance: "df-generated",
signature: "org.sql2o.Sql2o#Sql2o(String,String,String)",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "Sql2o",
methodParameters: "(String,String,String)",
},
},
};

View File

@@ -3,6 +3,7 @@ import * as React from "react";
import { ComponentMeta, ComponentStory } from "@storybook/react";
import { MethodRow as MethodRowComponent } from "../../view/data-extensions-editor/MethodRow";
import { CallClassification } from "../../data-extensions-editor/external-api-usage";
export default {
title: "Data Extensions Editor/Method Row",
@@ -23,6 +24,7 @@ MethodRow.args = {
methodName: "open",
methodParameters: "()",
supported: true,
supportedType: "summary",
usages: [
{
label: "open(...)",
@@ -33,6 +35,7 @@ MethodRow.args = {
endLine: 14,
endColumn: 35,
},
classification: CallClassification.Source,
},
{
label: "open(...)",
@@ -43,6 +46,7 @@ MethodRow.args = {
endLine: 25,
endColumn: 35,
},
classification: CallClassification.Source,
},
],
},
@@ -52,5 +56,10 @@ MethodRow.args = {
output: "ReturnValue",
kind: "taint",
provenance: "manual",
signature: "org.sql2o.Sql2o#open()",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "open",
methodParameters: "()",
},
};

View File

@@ -68,6 +68,7 @@ const ButtonsContainer = styled.div`
type Props = {
title: string;
libraryVersion?: string;
externalApiUsages: ExternalApiUsage[];
modeledMethods: Record<string, ModeledMethod>;
viewState: DataExtensionEditorViewState;
@@ -91,6 +92,7 @@ type Props = {
export const LibraryRow = ({
title,
libraryVersion,
externalApiUsages,
modeledMethods,
viewState,
@@ -158,7 +160,10 @@ export const LibraryRow = ({
<Codicon name="chevron-right" label="Expand" />
)}
<NameContainer>
<DependencyName>{title}</DependencyName>
<DependencyName>
{title}
{libraryVersion && <>@{libraryVersion}</>}
</DependencyName>
<ModeledPercentage>
{percentFormatter.format(modeledPercentage / 100)} modeled
</ModeledPercentage>

View File

@@ -0,0 +1,62 @@
import * as React from "react";
import { useMemo } from "react";
import {
CallClassification,
ExternalApiUsage,
} from "../../data-extensions-editor/external-api-usage";
import { VSCodeTag } from "@vscode/webview-ui-toolkit/react";
import styled from "styled-components";
const ClassificationsContainer = styled.div`
display: inline-flex;
flex-direction: row;
gap: 0.5rem;
`;
const ClassificationTag = styled(VSCodeTag)`
font-size: 0.75em;
`;
type Props = {
externalApiUsage: ExternalApiUsage;
};
export const MethodClassifications = ({ externalApiUsage }: Props) => {
const allUsageClassifications = useMemo(
() =>
new Set(
externalApiUsage.usages.map((usage) => {
return usage.classification;
}),
),
[externalApiUsage.usages],
);
const inSource = allUsageClassifications.has(CallClassification.Source);
const inTest = allUsageClassifications.has(CallClassification.Test);
const inGenerated = allUsageClassifications.has(CallClassification.Generated);
const tooltip = useMemo(() => {
if (inTest && inGenerated) {
return "This method is only used from test and generated code";
}
if (inTest) {
return "This method is only used from test code";
}
if (inGenerated) {
return "This method is only used from generated code";
}
return "";
}, [inTest, inGenerated]);
if (inSource) {
return null;
}
return (
<ClassificationsContainer title={tooltip}>
{inTest && <ClassificationTag>Test</ClassificationTag>}
{inGenerated && <ClassificationTag>Generated</ClassificationTag>}
</ClassificationsContainer>
);
};

View File

@@ -19,6 +19,7 @@ import { KindInput } from "./KindInput";
import { extensiblePredicateDefinitions } from "../../data-extensions-editor/predicates";
import { Mode } from "../../data-extensions-editor/shared/mode";
import { Dropdown } from "../common/Dropdown";
import { MethodClassifications } from "./MethodClassifications";
const ApiOrMethodCell = styled(VSCodeDataGridCell)`
display: flex;
@@ -39,7 +40,7 @@ const ViewLink = styled(VSCodeLink)`
white-space: nowrap;
`;
const modelTypeOptions = [
const modelTypeOptions: Array<{ value: ModeledMethodType; label: string }> = [
{ value: "none", label: "Unmodeled" },
{ value: "source", label: "Source" },
{ value: "sink", label: "Sink" },
@@ -100,6 +101,11 @@ function ModelableMethodRow(props: Props) {
...modeledMethod,
type: e.target.value as ModeledMethodType,
provenance: newProvenance,
signature: externalApiUsage.signature,
packageName: externalApiUsage.packageName,
typeName: externalApiUsage.typeName,
methodName: externalApiUsage.methodName,
methodParameters: externalApiUsage.methodParameters,
});
},
[onChange, externalApiUsage, modeledMethod, argumentsList],
@@ -197,6 +203,7 @@ function ModelableMethodRow(props: Props) {
</UsagesButton>
)}
<ViewLink onClick={jumpToUsage}>View</ViewLink>
<MethodClassifications externalApiUsage={externalApiUsage} />
</ApiOrMethodCell>
<VSCodeDataGridCell gridColumn={2}>
<Dropdown
@@ -233,10 +240,7 @@ function ModelableMethodRow(props: Props) {
);
}
function UnmodelableMethodRow(props: {
externalApiUsage: ExternalApiUsage;
mode: Mode;
}) {
function UnmodelableMethodRow(props: Props) {
const { externalApiUsage, mode } = props;
const jumpToUsage = useCallback(
@@ -255,9 +259,10 @@ function UnmodelableMethodRow(props: {
</UsagesButton>
)}
<ViewLink onClick={jumpToUsage}>View</ViewLink>
<MethodClassifications externalApiUsage={externalApiUsage} />
</ApiOrMethodCell>
<VSCodeDataGridCell gridColumn="span 4">
Method already modeled by CodeQL or a different extension pack
Method modeled by CodeQL or a different extension pack
</VSCodeDataGridCell>
</VSCodeDataGridRow>
);
@@ -266,8 +271,10 @@ function UnmodelableMethodRow(props: {
function ExternalApiUsageName(props: { externalApiUsage: ExternalApiUsage }) {
return (
<span>
{props.externalApiUsage.packageName}.{props.externalApiUsage.typeName}.
{props.externalApiUsage.methodName}
{props.externalApiUsage.packageName && (
<>{props.externalApiUsage.packageName}.</>
)}
{props.externalApiUsage.typeName}.{props.externalApiUsage.methodName}
{props.externalApiUsage.methodParameters}
</span>
);

View File

@@ -3,6 +3,7 @@ import { useMemo } from "react";
import { ExternalApiUsage } from "../../data-extensions-editor/external-api-usage";
import { ModeledMethod } from "../../data-extensions-editor/modeled-method";
import { LibraryRow } from "./LibraryRow";
import { Mode } from "../../data-extensions-editor/shared/mode";
import {
groupMethods,
sortGroupNames,
@@ -31,6 +32,10 @@ type Props = {
onGenerateFromSourceClick: () => void;
};
const libraryNameOverrides: Record<string, string> = {
rt: "Java Runtime",
};
export const ModeledMethodsList = ({
externalApiUsages,
unsavedModels,
@@ -46,6 +51,24 @@ export const ModeledMethodsList = ({
[externalApiUsages, viewState.mode],
);
const libraryVersions = useMemo(() => {
if (viewState.mode !== Mode.Application) {
return {};
}
const libraryVersions: Record<string, string> = {};
for (const externalApiUsage of externalApiUsages) {
const { library, libraryVersion } = externalApiUsage;
if (library && libraryVersion) {
libraryVersions[library] = libraryVersion;
}
}
return libraryVersions;
}, [externalApiUsages, viewState.mode]);
const sortedGroupNames = useMemo(() => sortGroupNames(grouped), [grouped]);
return (
@@ -53,7 +76,8 @@ export const ModeledMethodsList = ({
{sortedGroupNames.map((libraryName) => (
<LibraryRow
key={libraryName}
title={libraryName}
title={libraryNameOverrides[libraryName] ?? libraryName}
libraryVersion={libraryVersions[libraryName]}
externalApiUsages={grouped[libraryName]}
hasUnsavedChanges={unsavedModels.has(libraryName)}
modeledMethods={modeledMethods}

View File

@@ -1,4 +1,5 @@
[
"v2.14.0",
"v2.13.5",
"v2.12.7",
"v2.11.6",

View File

@@ -3,7 +3,10 @@ import {
createAutoModelRequest,
parsePredictedClassifications,
} from "../../../src/data-extensions-editor/auto-model";
import { ExternalApiUsage } from "../../../src/data-extensions-editor/external-api-usage";
import {
CallClassification,
ExternalApiUsage,
} from "../../../src/data-extensions-editor/external-api-usage";
import { ModeledMethod } from "../../../src/data-extensions-editor/modeled-method";
import {
ClassificationType,
@@ -22,6 +25,7 @@ describe("createAutoModelRequest", () => {
methodName: "run",
methodParameters: "(Class,String[])",
supported: false,
supportedType: "none",
usages: [
{
label: "run(...)",
@@ -32,6 +36,7 @@ describe("createAutoModelRequest", () => {
endLine: 9,
endColumn: 66,
},
classification: CallClassification.Source,
},
],
},
@@ -43,6 +48,7 @@ describe("createAutoModelRequest", () => {
methodName: "createQuery",
methodParameters: "(String)",
supported: false,
supportedType: "none",
usages: [
{
label: "createQuery(...)",
@@ -53,6 +59,7 @@ describe("createAutoModelRequest", () => {
endLine: 15,
endColumn: 56,
},
classification: CallClassification.Source,
},
{
label: "createQuery(...)",
@@ -63,6 +70,7 @@ describe("createAutoModelRequest", () => {
endLine: 26,
endColumn: 39,
},
classification: CallClassification.Source,
},
],
},
@@ -74,6 +82,7 @@ describe("createAutoModelRequest", () => {
methodName: "executeScalar",
methodParameters: "(Class)",
supported: false,
supportedType: "none",
usages: [
{
label: "executeScalar(...)",
@@ -84,6 +93,7 @@ describe("createAutoModelRequest", () => {
endLine: 15,
endColumn: 85,
},
classification: CallClassification.Source,
},
{
label: "executeScalar(...)",
@@ -94,6 +104,7 @@ describe("createAutoModelRequest", () => {
endLine: 26,
endColumn: 68,
},
classification: CallClassification.Source,
},
],
},
@@ -105,6 +116,7 @@ describe("createAutoModelRequest", () => {
methodName: "open",
methodParameters: "()",
supported: false,
supportedType: "none",
usages: [
{
label: "open(...)",
@@ -115,6 +127,7 @@ describe("createAutoModelRequest", () => {
endLine: 14,
endColumn: 35,
},
classification: CallClassification.Source,
},
{
label: "open(...)",
@@ -125,6 +138,7 @@ describe("createAutoModelRequest", () => {
endLine: 25,
endColumn: 35,
},
classification: CallClassification.Source,
},
],
},
@@ -136,6 +150,7 @@ describe("createAutoModelRequest", () => {
methodName: "println",
methodParameters: "(String)",
supported: false,
supportedType: "none",
usages: [
{
label: "println(...)",
@@ -146,6 +161,7 @@ describe("createAutoModelRequest", () => {
endLine: 29,
endColumn: 49,
},
classification: CallClassification.Source,
},
],
},
@@ -157,6 +173,7 @@ describe("createAutoModelRequest", () => {
methodName: "Sql2o",
methodParameters: "(String,String,String)",
supported: false,
supportedType: "none",
usages: [
{
label: "new Sql2o(...)",
@@ -167,6 +184,7 @@ describe("createAutoModelRequest", () => {
endLine: 10,
endColumn: 88,
},
classification: CallClassification.Source,
},
],
},
@@ -178,6 +196,7 @@ describe("createAutoModelRequest", () => {
methodName: "Sql2o",
methodParameters: "(String)",
supported: false,
supportedType: "none",
usages: [
{
label: "new Sql2o(...)",
@@ -188,6 +207,7 @@ describe("createAutoModelRequest", () => {
endLine: 23,
endColumn: 36,
},
classification: CallClassification.Source,
},
],
},
@@ -199,6 +219,7 @@ describe("createAutoModelRequest", () => {
methodName: "test",
methodParameters: "()",
supported: true,
supportedType: "neutral",
usages: [
{
label: "abc.test(...)",
@@ -209,6 +230,7 @@ describe("createAutoModelRequest", () => {
endLine: 23,
endColumn: 36,
},
classification: CallClassification.Source,
},
],
},
@@ -221,6 +243,11 @@ describe("createAutoModelRequest", () => {
input: "",
output: "",
provenance: "manual",
signature: "org.sql2o.Sql2o#open()",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "open",
methodParameters: "()",
},
"org.sql2o.Sql2o#Sql2o(String)": {
type: "sink",
@@ -228,6 +255,11 @@ describe("createAutoModelRequest", () => {
input: "Argument[0]",
output: "",
provenance: "manual",
signature: "org.sql2o.Sql2o#Sql2o(String)",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "Sql2o",
methodParameters: "(String)",
},
};
@@ -519,6 +551,11 @@ describe("parsePredictedClassifications", () => {
input: "Argument[0]",
output: "",
provenance: "ai-generated",
signature: "org.sql2o.Sql2o#createQuery(String)",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "createQuery",
methodParameters: "(String)",
},
"org.sql2o.Sql2o#executeScalar(Class)": {
type: "neutral",
@@ -526,6 +563,11 @@ describe("parsePredictedClassifications", () => {
input: "",
output: "",
provenance: "ai-generated",
signature: "org.sql2o.Sql2o#executeScalar(Class)",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "executeScalar",
methodParameters: "(Class)",
},
"org.sql2o.Sql2o#Sql2o(String,String,String)": {
type: "sink",
@@ -533,6 +575,11 @@ describe("parsePredictedClassifications", () => {
input: "Argument[1]",
output: "",
provenance: "ai-generated",
signature: "org.sql2o.Sql2o#Sql2o(String,String,String)",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "Sql2o",
methodParameters: "(String,String,String)",
},
});
});

View File

@@ -1,5 +1,6 @@
import { decodeBqrsToExternalApiUsages } from "../../../src/data-extensions-editor/bqrs";
import { DecodedBqrsChunk } from "../../../src/common/bqrs-cli-types";
import { CallClassification } from "../../../src/data-extensions-editor/external-api-usage";
describe("decodeBqrsToExternalApiUsages", () => {
const chunk: DecodedBqrsChunk = {
@@ -8,6 +9,12 @@ describe("decodeBqrsToExternalApiUsages", () => {
{ name: "apiName", kind: "String" },
{ kind: "String" },
{ kind: "String" },
{ kind: "String" },
{ kind: "String" },
{ name: "type", kind: "String" },
{ kind: "String" },
{ name: "classification", kind: "String" },
{ kind: "String" },
],
tuples: [
[
@@ -24,6 +31,12 @@ describe("decodeBqrsToExternalApiUsages", () => {
"java.io.PrintStream#println(String)",
"true",
"supported",
"rt.jar",
"",
"sink",
"type",
"source",
"classification",
],
[
{
@@ -39,6 +52,12 @@ describe("decodeBqrsToExternalApiUsages", () => {
"org.springframework.boot.SpringApplication#run(Class,String[])",
"false",
"supported",
"spring-boot-3.0.2.jar",
"",
"none",
"type",
"source",
"classification",
],
[
{
@@ -54,6 +73,12 @@ describe("decodeBqrsToExternalApiUsages", () => {
"org.sql2o.Connection#createQuery(String)",
"true",
"supported",
"sql2o-1.6.0.jar",
"",
"sink",
"type",
"source",
"classification",
],
[
{
@@ -69,6 +94,12 @@ describe("decodeBqrsToExternalApiUsages", () => {
"org.sql2o.Connection#createQuery(String)",
"true",
"supported",
"sql2o-1.6.0.jar",
"",
"sink",
"type",
"source",
"classification",
],
[
{
@@ -84,6 +115,12 @@ describe("decodeBqrsToExternalApiUsages", () => {
"org.sql2o.Query#executeScalar(Class)",
"true",
"supported",
"sql2o-1.6.0.jar",
"",
"sink",
"type",
"source",
"classification",
],
[
{
@@ -99,6 +136,12 @@ describe("decodeBqrsToExternalApiUsages", () => {
"org.sql2o.Query#executeScalar(Class)",
"true",
"supported",
"sql2o-1.6.0.jar",
"",
"sink",
"type",
"source",
"classification",
],
[
{
@@ -114,6 +157,12 @@ describe("decodeBqrsToExternalApiUsages", () => {
"org.sql2o.Sql2o#open()",
"true",
"supported",
"sql2o-1.6.0.jar",
"",
"sink",
"type",
"source",
"classification",
],
[
{
@@ -129,6 +178,12 @@ describe("decodeBqrsToExternalApiUsages", () => {
"org.sql2o.Sql2o#open()",
"true",
"supported",
"sql2o-1.6.0.jar",
"",
"sink",
"type",
"source",
"classification",
],
[
{
@@ -144,6 +199,12 @@ describe("decodeBqrsToExternalApiUsages", () => {
"org.sql2o.Sql2o#Sql2o(String,String,String)",
"true",
"supported",
"sql2o-1.6.0.jar",
"",
"sink",
"type",
"source",
"classification",
],
[
{
@@ -159,6 +220,12 @@ describe("decodeBqrsToExternalApiUsages", () => {
"org.sql2o.Sql2o#Sql2o(String)",
"true",
"supported",
"sql2o-1.6.0.jar",
"",
"sink",
"type",
"source",
"classification",
],
],
};
@@ -169,12 +236,15 @@ describe("decodeBqrsToExternalApiUsages", () => {
// - Sorting the array of usages is guaranteed to be a stable sort
expect(decodeBqrsToExternalApiUsages(chunk)).toEqual([
{
library: "rt",
libraryVersion: undefined,
signature: "java.io.PrintStream#println(String)",
packageName: "java.io",
typeName: "PrintStream",
methodName: "println",
methodParameters: "(String)",
supported: true,
supportedType: "sink",
usages: [
{
label: "println(...)",
@@ -185,10 +255,13 @@ describe("decodeBqrsToExternalApiUsages", () => {
endLine: 29,
endColumn: 49,
},
classification: CallClassification.Source,
},
],
},
{
library: "spring-boot",
libraryVersion: "3.0.2",
signature:
"org.springframework.boot.SpringApplication#run(Class,String[])",
packageName: "org.springframework.boot",
@@ -196,6 +269,7 @@ describe("decodeBqrsToExternalApiUsages", () => {
methodName: "run",
methodParameters: "(Class,String[])",
supported: false,
supportedType: "none",
usages: [
{
label: "run(...)",
@@ -206,16 +280,20 @@ describe("decodeBqrsToExternalApiUsages", () => {
endLine: 9,
endColumn: 66,
},
classification: CallClassification.Source,
},
],
},
{
library: "sql2o",
libraryVersion: "1.6.0",
signature: "org.sql2o.Connection#createQuery(String)",
packageName: "org.sql2o",
typeName: "Connection",
methodName: "createQuery",
methodParameters: "(String)",
supported: true,
supportedType: "sink",
usages: [
{
label: "createQuery(...)",
@@ -226,6 +304,7 @@ describe("decodeBqrsToExternalApiUsages", () => {
endLine: 15,
endColumn: 56,
},
classification: CallClassification.Source,
},
{
label: "createQuery(...)",
@@ -236,16 +315,20 @@ describe("decodeBqrsToExternalApiUsages", () => {
endLine: 26,
endColumn: 39,
},
classification: CallClassification.Source,
},
],
},
{
library: "sql2o",
libraryVersion: "1.6.0",
signature: "org.sql2o.Query#executeScalar(Class)",
packageName: "org.sql2o",
typeName: "Query",
methodName: "executeScalar",
methodParameters: "(Class)",
supported: true,
supportedType: "sink",
usages: [
{
label: "executeScalar(...)",
@@ -256,6 +339,7 @@ describe("decodeBqrsToExternalApiUsages", () => {
endLine: 15,
endColumn: 85,
},
classification: CallClassification.Source,
},
{
label: "executeScalar(...)",
@@ -266,16 +350,20 @@ describe("decodeBqrsToExternalApiUsages", () => {
endLine: 26,
endColumn: 68,
},
classification: CallClassification.Source,
},
],
},
{
library: "sql2o",
libraryVersion: "1.6.0",
signature: "org.sql2o.Sql2o#open()",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "open",
methodParameters: "()",
supported: true,
supportedType: "sink",
usages: [
{
label: "open(...)",
@@ -286,6 +374,7 @@ describe("decodeBqrsToExternalApiUsages", () => {
endLine: 14,
endColumn: 35,
},
classification: CallClassification.Source,
},
{
label: "open(...)",
@@ -296,16 +385,20 @@ describe("decodeBqrsToExternalApiUsages", () => {
endLine: 25,
endColumn: 35,
},
classification: CallClassification.Source,
},
],
},
{
library: "sql2o",
libraryVersion: "1.6.0",
signature: "org.sql2o.Sql2o#Sql2o(String,String,String)",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "Sql2o",
methodParameters: "(String,String,String)",
supported: true,
supportedType: "sink",
usages: [
{
label: "new Sql2o(...)",
@@ -316,16 +409,20 @@ describe("decodeBqrsToExternalApiUsages", () => {
endLine: 10,
endColumn: 88,
},
classification: CallClassification.Source,
},
],
},
{
library: "sql2o",
libraryVersion: "1.6.0",
signature: "org.sql2o.Sql2o#Sql2o(String)",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "Sql2o",
methodParameters: "(String)",
supported: true,
supportedType: "sink",
usages: [
{
label: "new Sql2o(...)",
@@ -336,6 +433,7 @@ describe("decodeBqrsToExternalApiUsages", () => {
endLine: 23,
endColumn: 36,
},
classification: CallClassification.Source,
},
],
},

View File

@@ -0,0 +1,99 @@
import { parseLibraryFilename } from "../../../src/data-extensions-editor/library";
describe("parseLibraryFilename", () => {
const testCases = [
{ filename: "sql2o-1.6.0.jar", name: "sql2o", version: "1.6.0" },
{
filename: "spring-boot-3.0.2.jar",
name: "spring-boot",
version: "3.0.2",
},
{ filename: "rt.jar", name: "rt", version: undefined },
{ filename: "guava-15.0.jar", name: "guava", version: "15.0" },
{
filename: "embedded-db-junit-1.0.0.jar",
name: "embedded-db-junit",
version: "1.0.0",
},
{
filename: "h2-1.3.160.jar",
name: "h2",
version: "1.3.160",
},
{
filename: "joda-time-2.0.jar",
name: "joda-time",
version: "2.0",
},
{
filename: "System.Runtime.dll",
name: "System.Runtime",
version: undefined,
},
{
filename: "System.Linq.Expressions.dll",
name: "System.Linq.Expressions",
version: undefined,
},
{
filename: "System.Diagnostics.Debug.dll",
name: "System.Diagnostics.Debug",
version: undefined,
},
{
filename: "spring-boot-3.1.0-rc2.jar",
name: "spring-boot",
version: "3.1.0-rc2",
},
{
filename: "org.eclipse.sisu.plexus-0.9.0.M2.jar",
name: "org.eclipse.sisu.plexus",
version: "0.9.0.M2",
},
{
filename: "org.eclipse.sisu.inject-0.9.0.M2.jar",
name: "org.eclipse.sisu.inject",
version: "0.9.0.M2",
},
{
filename: "slf4j-api-1.7.36.jar",
name: "slf4j-api",
version: "1.7.36",
},
{
filename: "guava-30.1.1-jre.jar",
name: "guava",
version: "30.1.1-jre",
},
{
filename: "caliper-1.0-beta-3.jar",
name: "caliper",
version: "1.0-beta-3",
},
{
filename: "protobuf-java-4.0.0-rc-2.jar",
name: "protobuf-java",
version: "4.0.0-rc-2",
},
{
filename: "jetty-util-9.4.51.v20230217.jar",
name: "jetty-util",
version: "9.4.51.v20230217",
},
{
filename: "jetty-servlet-9.4.51.v20230217.jar",
name: "jetty-servlet",
version: "9.4.51.v20230217",
},
];
test.each(testCases)(
"$filename is $name@$version",
({ filename, name, version }) => {
expect(parseLibraryFilename(filename)).toEqual({
name,
version,
});
},
);
});

View File

@@ -5,82 +5,22 @@ import {
createFilenameForLibrary,
loadDataExtensionYaml,
} from "../../../src/data-extensions-editor/yaml";
import { CallClassification } from "../../../src/data-extensions-editor/external-api-usage";
describe("createDataExtensionYaml", () => {
it("creates the correct YAML file", () => {
const yaml = createDataExtensionYaml("java", [
{
externalApiUsage: {
library: "sql2o-1.6.0.jar",
signature: "org.sql2o.Connection#createQuery(String)",
packageName: "org.sql2o",
typeName: "Connection",
methodName: "createQuery",
methodParameters: "(String)",
supported: true,
usages: [
{
label: "createQuery(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 15,
startColumn: 13,
endLine: 15,
endColumn: 56,
},
},
{
label: "createQuery(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 26,
startColumn: 13,
endLine: 26,
endColumn: 39,
},
},
],
},
modeledMethod: {
type: "sink",
input: "Argument[0]",
output: "",
kind: "sql",
provenance: "df-generated",
},
},
{
externalApiUsage: {
library: "sql2o-1.6.0.jar",
signature: "org.sql2o.Query#executeScalar(Class)",
packageName: "org.sql2o",
typeName: "Query",
methodName: "executeScalar",
methodParameters: "(Class)",
supported: true,
usages: [
{
label: "executeScalar(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 15,
startColumn: 13,
endLine: 15,
endColumn: 85,
},
},
{
label: "executeScalar(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 26,
startColumn: 13,
endLine: 26,
endColumn: 68,
},
},
],
},
type: "sink",
input: "Argument[0]",
output: "",
kind: "sql",
provenance: "df-generated",
signature: "org.sql2o.Connection#createQuery(String)",
packageName: "org.sql2o",
typeName: "Connection",
methodName: "createQuery",
methodParameters: "(String)",
},
]);
@@ -141,13 +81,15 @@ describe("createDataExtensionYamlsForApplicationMode", () => {
"java",
[
{
library: "sql2o-1.6.0.jar",
library: "sql2o",
libraryVersion: "1.6.0",
signature: "org.sql2o.Connection#createQuery(String)",
packageName: "org.sql2o",
typeName: "Connection",
methodName: "createQuery",
methodParameters: "(String)",
supported: true,
supportedType: "sink",
usages: [
{
label: "createQuery(...)",
@@ -158,6 +100,7 @@ describe("createDataExtensionYamlsForApplicationMode", () => {
endLine: 15,
endColumn: 56,
},
classification: CallClassification.Source,
},
{
label: "createQuery(...)",
@@ -168,17 +111,20 @@ describe("createDataExtensionYamlsForApplicationMode", () => {
endLine: 26,
endColumn: 39,
},
classification: CallClassification.Source,
},
],
},
{
library: "sql2o-1.6.0.jar",
library: "sql2o",
libraryVersion: "1.6.0",
signature: "org.sql2o.Query#executeScalar(Class)",
packageName: "org.sql2o",
typeName: "Query",
methodName: "executeScalar",
methodParameters: "(Class)",
supported: true,
supportedType: "neutral",
usages: [
{
label: "executeScalar(...)",
@@ -189,6 +135,7 @@ describe("createDataExtensionYamlsForApplicationMode", () => {
endLine: 15,
endColumn: 85,
},
classification: CallClassification.Source,
},
{
label: "executeScalar(...)",
@@ -199,17 +146,20 @@ describe("createDataExtensionYamlsForApplicationMode", () => {
endLine: 26,
endColumn: 68,
},
classification: CallClassification.Source,
},
],
},
{
library: "sql2o-2.5.0-alpha1.jar",
library: "sql2o",
libraryVersion: "2.5.0-alpha1",
signature: "org.sql2o.Sql2o#Sql2o(String,String,String)",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "Sql2o",
methodParameters: "(String,String,String)",
supported: false,
supportedType: "none",
usages: [
{
label: "new Sql2o(...)",
@@ -220,11 +170,13 @@ describe("createDataExtensionYamlsForApplicationMode", () => {
endLine: 10,
endColumn: 88,
},
classification: CallClassification.Source,
},
],
},
{
library: "spring-boot-3.0.2.jar",
library: "spring-boot",
libraryVersion: "3.0.2",
signature:
"org.springframework.boot.SpringApplication#run(Class,String[])",
packageName: "org.springframework.boot",
@@ -232,6 +184,7 @@ describe("createDataExtensionYamlsForApplicationMode", () => {
methodName: "run",
methodParameters: "(Class,String[])",
supported: false,
supportedType: "none",
usages: [
{
label: "run(...)",
@@ -242,17 +195,19 @@ describe("createDataExtensionYamlsForApplicationMode", () => {
endLine: 9,
endColumn: 66,
},
classification: CallClassification.Source,
},
],
},
{
library: "rt.jar",
library: "rt",
signature: "java.io.PrintStream#println(String)",
packageName: "java.io",
typeName: "PrintStream",
methodName: "println",
methodParameters: "(String)",
supported: true,
supportedType: "summary",
usages: [
{
label: "println(...)",
@@ -263,6 +218,7 @@ describe("createDataExtensionYamlsForApplicationMode", () => {
endLine: 29,
endColumn: 49,
},
classification: CallClassification.Source,
},
],
},
@@ -274,6 +230,11 @@ describe("createDataExtensionYamlsForApplicationMode", () => {
output: "",
kind: "sql",
provenance: "df-generated",
signature: "org.sql2o.Connection#createQuery(String)",
packageName: "org.sql2o",
typeName: "Connection",
methodName: "createQuery",
methodParameters: "(String)",
},
"org.springframework.boot.SpringApplication#run(Class,String[])": {
type: "neutral",
@@ -281,6 +242,12 @@ describe("createDataExtensionYamlsForApplicationMode", () => {
output: "",
kind: "summary",
provenance: "manual",
signature:
"org.springframework.boot.SpringApplication#run(Class,String[])",
packageName: "org.springframework.boot",
typeName: "SpringApplication",
methodName: "run",
methodParameters: "(Class,String[])",
},
"org.sql2o.Sql2o#Sql2o(String,String,String)": {
type: "sink",
@@ -288,6 +255,11 @@ describe("createDataExtensionYamlsForApplicationMode", () => {
output: "",
kind: "jndi",
provenance: "manual",
signature: "org.sql2o.Sql2o#Sql2o(String,String,String)",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "Sql2o",
methodParameters: "(String,String,String)",
},
},
);
@@ -356,6 +328,7 @@ describe("createDataExtensionYamlsForFrameworkMode", () => {
methodName: "createQuery",
methodParameters: "(String)",
supported: true,
supportedType: "sink",
usages: [
{
label: "createQuery(...)",
@@ -366,6 +339,7 @@ describe("createDataExtensionYamlsForFrameworkMode", () => {
endLine: 15,
endColumn: 56,
},
classification: CallClassification.Source,
},
{
label: "createQuery(...)",
@@ -376,6 +350,7 @@ describe("createDataExtensionYamlsForFrameworkMode", () => {
endLine: 26,
endColumn: 39,
},
classification: CallClassification.Source,
},
],
},
@@ -387,6 +362,7 @@ describe("createDataExtensionYamlsForFrameworkMode", () => {
methodName: "executeScalar",
methodParameters: "(Class)",
supported: true,
supportedType: "neutral",
usages: [
{
label: "executeScalar(...)",
@@ -397,6 +373,7 @@ describe("createDataExtensionYamlsForFrameworkMode", () => {
endLine: 15,
endColumn: 85,
},
classification: CallClassification.Source,
},
{
label: "executeScalar(...)",
@@ -407,6 +384,7 @@ describe("createDataExtensionYamlsForFrameworkMode", () => {
endLine: 26,
endColumn: 68,
},
classification: CallClassification.Source,
},
],
},
@@ -418,6 +396,7 @@ describe("createDataExtensionYamlsForFrameworkMode", () => {
methodName: "Sql2o",
methodParameters: "(String,String,String)",
supported: false,
supportedType: "none",
usages: [
{
label: "new Sql2o(...)",
@@ -428,6 +407,7 @@ describe("createDataExtensionYamlsForFrameworkMode", () => {
endLine: 10,
endColumn: 88,
},
classification: CallClassification.Source,
},
],
},
@@ -439,6 +419,11 @@ describe("createDataExtensionYamlsForFrameworkMode", () => {
output: "",
kind: "sql",
provenance: "df-generated",
signature: "org.sql2o.Connection#createQuery(String)",
packageName: "org.sql2o",
typeName: "Connection",
methodName: "createQuery",
methodParameters: "(String)",
},
"org.sql2o.Sql2o#Sql2o(String,String,String)": {
type: "sink",
@@ -446,6 +431,11 @@ describe("createDataExtensionYamlsForFrameworkMode", () => {
output: "",
kind: "jndi",
provenance: "manual",
signature: "org.sql2o.Sql2o#Sql2o(String,String,String)",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "Sql2o",
methodParameters: "(String,String,String)",
},
},
);
@@ -520,6 +510,11 @@ describe("loadDataExtensionYaml", () => {
output: "",
type: "sink",
provenance: "manual",
signature: "org.sql2o.Connection#createQuery(String)",
packageName: "org.sql2o",
typeName: "Connection",
methodName: "createQuery",
methodParameters: "(String)",
},
});
});
@@ -539,37 +534,28 @@ describe("loadDataExtensionYaml", () => {
describe("createFilenameForLibrary", () => {
const testCases = [
{ library: "sql2o.jar", filename: "models/sql2o.model.yml" },
{
library: "sql2o-1.6.0.jar",
library: "sql2o",
filename: "models/sql2o.model.yml",
},
{
library: "spring-boot-3.0.2.jar",
library: "spring-boot",
filename: "models/spring-boot.model.yml",
},
{
library: "spring-boot-v3.0.2.jar",
library: "spring--boot",
filename: "models/spring-boot.model.yml",
},
{
library: "spring-boot-3.0.2-alpha1.jar",
filename: "models/spring-boot.model.yml",
},
{
library: "spring-boot-3.0.2beta2.jar",
filename: "models/spring-boot.model.yml",
},
{
library: "rt.jar",
library: "rt",
filename: "models/rt.model.yml",
},
{
library: "System.Runtime.dll",
library: "System.Runtime",
filename: "models/system.runtime.model.yml",
},
{
library: "System.Runtime.1.5.0.dll",
library: "System..Runtime",
filename: "models/system.runtime.model.yml",
},
];