mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #2535 from github/koesie10/framework-mode

Browse Source 
Add initial implementation of framework mode
This commit is contained in:
Koen Vlaswinkel
2023-06-23 13:57:09 +02:00
committed by GitHub
parent 82cdf03d8c 4f367119cb
commit a76bd4627c
20 changed files with 654 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
extensions/ql-vscode/src/common/interface-types.ts
View File

@@ -20,6 +20,7 @@ import { DataFlowPaths } from "../variant-analysis/shared/data-flow-paths";
import { ExternalApiUsage } from "../data-extensions-editor/external-api-usage";
import { ModeledMethod } from "../data-extensions-editor/modeled-method";
import { DataExtensionEditorViewState } from "../data-extensions-editor/shared/view-state";
import { Mode } from "../data-extensions-editor/shared/mode";
/**
* This module contains types and code that are shared between
@@ -521,6 +522,11 @@ export interface AddModeledMethodsMessage {
overrideNone?: boolean;
}
export interface SwitchModeMessage {
t: "switchMode";
mode: Mode;
}
export interface JumpToUsageMessage {
t: "jumpToUsage";
location: ResolvableLocationValue;
@@ -559,6 +565,7 @@ export type ToDataExtensionsEditorMessage =
export type FromDataExtensionsEditorMessage =
| ViewLoadedMsg
| SwitchModeMessage
| OpenModelFileMessage
| OpenExtensionPackMessage
| JumpToUsageMessage

5
extensions/ql-vscode/src/config.ts
View File

@@ -714,6 +714,7 @@ export function showQueriesPanel(): boolean {
const DATA_EXTENSIONS = new Setting("dataExtensions", ROOT_SETTING);
const LLM_GENERATION = new Setting("llmGeneration", DATA_EXTENSIONS);
const FRAMEWORK_MODE = new Setting("frameworkMode", DATA_EXTENSIONS);
const DISABLE_AUTO_NAME_EXTENSION_PACK = new Setting(
"disableAutoNameExtensionPack",
DATA_EXTENSIONS,
@@ -723,6 +724,10 @@ export function showLlmGeneration(): boolean {
return !!LLM_GENERATION.getValue<boolean>();
}
export function enableFrameworkMode(): boolean {
return !!FRAMEWORK_MODE.getValue<boolean>();
}
export function disableAutoNameExtensionPack(): boolean {
return !!DISABLE_AUTO_NAME_EXTENSION_PACK.getValue<boolean>();
}

2
extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts
View File

@@ -32,7 +32,7 @@ export async function getAutoModelUsages({
// This will re-run the query that was already run when opening the data extensions editor. This
// might be unnecessary, but this makes it really easy to get the path to the BQRS file which we
// need to interpret the results.
const queryResult = await runQuery({
const queryResult = await runQuery("applicationModeQuery", {
cliServer,
queryRunner,
queryStorageDir,

57
extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts
View File

@@ -36,7 +36,8 @@ import { decodeBqrsToExternalApiUsages } from "./bqrs";
import { redactableError } from "../common/errors";
import { readQueryResults, runQuery } from "./external-api-usage-query";
import {
createDataExtensionYamlsPerLibrary,
createDataExtensionYamlsForApplicationMode,
createDataExtensionYamlsForFrameworkMode,
createFilenameForLibrary,
loadDataExtensionYaml,
} from "./yaml";
@@ -48,9 +49,10 @@ import {
createAutoModelRequest,
parsePredictedClassifications,
} from "./auto-model";
import { showLlmGeneration } from "../config";
import { enableFrameworkMode, showLlmGeneration } from "../config";
import { getAutoModelUsages } from "./auto-model-usages-query";
import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
import { Mode } from "./shared/mode";
export class DataExtensionsEditorView extends AbstractWebview<
ToDataExtensionsEditorMessage,
@@ -65,6 +67,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
private readonly queryStorageDir: string,
private readonly databaseItem: DatabaseItem,
private readonly extensionPack: ExtensionPack,
private mode: Mode = Mode.Application,
) {
super(ctx);
}
@@ -138,6 +141,12 @@ export class DataExtensionsEditorView extends AbstractWebview<
msg.modeledMethods,
);
break;
case "switchMode":
this.mode = msg.mode;
await Promise.all([this.setViewState(), this.loadExternalApiUsages()]);
break;
default:
assertNever(msg);
@@ -159,7 +168,9 @@ export class DataExtensionsEditorView extends AbstractWebview<
t: "setDataExtensionEditorViewState",
viewState: {
extensionPack: this.extensionPack,
enableFrameworkMode: enableFrameworkMode(),
showLlmButton: showLlmGeneration(),
mode: this.mode,
},
});
}
@@ -188,11 +199,26 @@ export class DataExtensionsEditorView extends AbstractWebview<
externalApiUsages: ExternalApiUsage[],
modeledMethods: Record<string, ModeledMethod>,
): Promise<void> {
const yamls = createDataExtensionYamlsPerLibrary(
let yamls: Record<string, string>;
switch (this.mode) {
case Mode.Application:
yamls = createDataExtensionYamlsForApplicationMode(
this.databaseItem.language,
externalApiUsages,
modeledMethods,
);
break;
case Mode.Framework:
yamls = createDataExtensionYamlsForFrameworkMode(
this.databaseItem.name,
this.databaseItem.language,
externalApiUsages,
modeledMethods,
);
break;
default:
assertNever(this.mode);
}
for (const [filename, yaml] of Object.entries(yamls)) {
await outputFile(join(this.extensionPack.path, filename), yaml);
@@ -255,7 +281,11 @@ export class DataExtensionsEditorView extends AbstractWebview<
const cancellationTokenSource = new CancellationTokenSource();
try {
const queryResult = await runQuery({
const queryResult = await runQuery(
this.mode === Mode.Framework
? "frameworkModeQuery"
: "applicationModeQuery",
{
cliServer: this.cliServer,
queryRunner: this.queryRunner,
databaseItem: this.databaseItem,
@@ -264,7 +294,8 @@ export class DataExtensionsEditorView extends AbstractWebview<
void this.showProgress(progressUpdate, 1500);
},
token: cancellationTokenSource.token,
});
},
);
if (!queryResult) {
await this.clearProgress();
return;
@@ -313,12 +344,17 @@ export class DataExtensionsEditorView extends AbstractWebview<
protected async generateModeledMethods(): Promise<void> {
const tokenSource = new CancellationTokenSource();
let addedDatabase: DatabaseItem | undefined;
// In application mode, we need the database of a specific library to generate
// the modeled methods. In framework mode, we'll use the current database.
if (this.mode === Mode.Application) {
const selectedDatabase = this.databaseManager.currentDatabaseItem;
// The external API methods are in the library source code, so we need to ask
// the user to import the library database. We need to have the database
// imported to the query server, so we need to register it to our workspace.
const database = await promptImportGithubDatabase(
addedDatabase = await promptImportGithubDatabase(
this.app.commands,
this.databaseManager,
this.app.workspaceStoragePath ?? this.app.globalStoragePath,
@@ -326,7 +362,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
(update) => this.showProgress(update),
this.cliServer,
);
if (!database) {
if (!addedDatabase) {
await this.clearProgress();
void this.app.logger.log("No database chosen");
@@ -336,6 +372,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
// The library database was set as the current database by importing it,
// but we need to set it back to the originally selected database.
await this.databaseManager.setCurrentDatabaseItem(selectedDatabase);
}
await this.showProgress({
step: 0,
@@ -348,7 +385,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
cliServer: this.cliServer,
queryRunner: this.queryRunner,
queryStorageDir: this.queryStorageDir,
databaseItem: database,
databaseItem: addedDatabase ?? this.databaseItem,
onResults: async (results) => {
const modeledMethodsByName: Record<string, ModeledMethod> = {};
@@ -375,6 +412,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
);
}
if (addedDatabase) {
// After the flow model has been generated, we can remove the temporary database
// which we used for generating the flow model.
await this.showProgress({
@@ -382,7 +420,8 @@ export class DataExtensionsEditorView extends AbstractWebview<
maxStep: 4000,
message: "Removing temporary database",
});
await this.databaseManager.removeDatabaseItem(database);
await this.databaseManager.removeDatabaseItem(addedDatabase);
}
await this.clearProgress();
}

2
extensions/ql-vscode/src/data-extensions-editor/extension-pack-name.ts
View File

@@ -37,7 +37,7 @@ export function autoNameExtensionPack(
};
}
function sanitizeExtensionPackName(name: string) {
export function sanitizeExtensionPackName(name: string) {
// Lowercase everything
name = name.toLowerCase();

10
extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts
View File

@@ -15,6 +15,7 @@ import { QueryResultType } from "../query-server/new-messages";
import { join } from "path";
import { redactableError } from "../common/errors";
import { telemetryListener } from "../common/vscode/telemetry";
import { Query } from "./queries/query";
export type RunQueryOptions = {
cliServer: Pick<CodeQLCliServer, "resolveQlpacks">;
@@ -26,14 +27,17 @@ export type RunQueryOptions = {
token: CancellationToken;
};
export async function runQuery({
export async function runQuery(
queryName: keyof Omit<Query, "dependencies">,
{
cliServer,
queryRunner,
databaseItem,
queryStorageDir,
progress,
token,
}: RunQueryOptions): Promise<CoreCompletedQuery | undefined> {
}: RunQueryOptions,
): Promise<CoreCompletedQuery | undefined> {
// The below code is temporary to allow for rapid prototyping of the queries. Once the queries are stabilized, we will
// move these queries into the `github/codeql` repository and use them like any other contextual (e.g. AST) queries.
// This is intentionally not pretty code, as it will be removed soon.
@@ -61,7 +65,7 @@ export async function runQuery({
const queryDir = (await dir({ unsafeCleanup: true })).path;
const queryFile = join(queryDir, "FetchExternalApis.ql");
await writeFile(queryFile, query.mainQuery, "utf8");
await writeFile(queryFile, query[queryName], "utf8");
if (query.dependencies) {
for (const [filename, contents] of Object.entries(query.dependencies)) {

135
extensions/ql-vscode/src/data-extensions-editor/queries/csharp.ts
View File

@@ -1,7 +1,7 @@
import { Query } from "./query";
export const fetchExternalApisQuery: Query = {
mainQuery: `/**
applicationModeQuery: `/**
* @name Usage of APIs coming from external libraries
* @description A list of 3rd party APIs used in the codebase.
* @tags telemetry
@@ -27,6 +27,139 @@ where
supported = isSupported(api) and
usage = aUsage(api)
select usage, apiName, supported.toString(), "supported", api.getFile().getBaseName(), "library"
`,
frameworkModeQuery: `/**
* @name Usage of APIs coming from external libraries
* @description A list of 3rd party APIs used in the codebase.
* @tags telemetry
* @kind problem
* @id cs/telemetry/fetch-external-apis
*/
private import csharp
private import dotnet
private import semmle.code.csharp.dispatch.Dispatch
private import semmle.code.csharp.dataflow.ExternalFlow
private import semmle.code.csharp.dataflow.FlowSummary
private import semmle.code.csharp.dataflow.internal.DataFlowImplCommon as DataFlowImplCommon
private import semmle.code.csharp.dataflow.internal.DataFlowPrivate
private import semmle.code.csharp.dataflow.internal.DataFlowDispatch as DataFlowDispatch
private import semmle.code.csharp.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
private import semmle.code.csharp.dataflow.internal.TaintTrackingPrivate
private import semmle.code.csharp.security.dataflow.flowsources.Remote
pragma[nomagic]
private predicate isTestNamespace(Namespace ns) {
ns.getFullName()
.matches([
"NUnit.Framework%", "Xunit%", "Microsoft.VisualStudio.TestTools.UnitTesting%", "Moq%"
])
}
/**
* A test library.
*/
class TestLibrary extends RefType {
TestLibrary() { isTestNamespace(this.getNamespace()) }
}
/** Holds if the given callable is not worth supporting. */
private predicate isUninteresting(DotNet::Callable c) {
c.getDeclaringType() instanceof TestLibrary or
c.(Constructor).isParameterless()
}
class PublicMethod extends DotNet::Member {
PublicMethod() { this.isPublic() and not isUninteresting(this) and exists(this.(DotNet::Member)) }
/**
* Gets the unbound type, name and parameter types of this API.
*/
bindingset[this]
private string getSignature() {
result =
this.getDeclaringType().getUnboundDeclaration() + "." + this.getName() + "(" +
parameterQualifiedTypeNamesToString(this) + ")"
}
/**
* Gets the namespace of this API.
*/
bindingset[this]
string getNamespace() { this.getDeclaringType().hasQualifiedName(result, _) }
/**
* Gets the namespace and signature of this API.
*/
bindingset[this]
string getApiName() { result = this.getNamespace() + "#" + this.getSignature() }
/** Gets a node that is an input to a call to this API. */
private ArgumentNode getAnInput() {
result
.getCall()
.(DataFlowDispatch::NonDelegateDataFlowCall)
.getATarget(_)
.getUnboundDeclaration() = this
}
/** Gets a node that is an output from a call to this API. */
private DataFlow::Node getAnOutput() {
exists(
Call c, DataFlowDispatch::NonDelegateDataFlowCall dc, DataFlowImplCommon::ReturnKindExt ret
|
dc.getDispatchCall().getCall() = c and
c.getTarget().getUnboundDeclaration() = this
|
result = ret.getAnOutNode(dc)
)
}
/** Holds if this API has a supported summary. */
pragma[nomagic]
predicate hasSummary() {
this instanceof SummarizedCallable
or
defaultAdditionalTaintStep(this.getAnInput(), _)
}
/** Holds if this API is a known source. */
pragma[nomagic]
predicate isSource() {
this.getAnOutput() instanceof RemoteFlowSource or sourceNode(this.getAnOutput(), _)
}
/** Holds if this API is a known sink. */
pragma[nomagic]
predicate isSink() { sinkNode(this.getAnInput(), _) }
/** Holds if this API is a known neutral. */
pragma[nomagic]
predicate isNeutral() { this instanceof FlowSummaryImpl::Public::NeutralCallable }
/**
* Holds if this API is supported by existing CodeQL libraries, that is, it is either a
* recognized source, sink or neutral or it has a flow summary.
*/
predicate isSupported() {
this.hasSummary() or this.isSource() or this.isSink() or this.isNeutral()
}
}
private boolean isSupported(PublicMethod publicMethod) {
publicMethod.isSupported() and result = true
or
not publicMethod.isSupported() and
result = false
}
from PublicMethod publicMethod, string apiName, boolean supported
where
apiName = publicMethod.getApiName() and
publicMethod.getDeclaringType().fromSource() and
supported = isSupported(publicMethod)
select publicMethod, apiName, supported.toString(), "supported",
publicMethod.getFile().getBaseName(), "library"
`,
dependencies: {
"ExternalApi.qll": `/** Provides classes and predicates related to handling APIs from external libraries. */

96
extensions/ql-vscode/src/data-extensions-editor/queries/java.ts
View File

@@ -1,7 +1,7 @@
import { Query } from "./query";
export const fetchExternalApisQuery: Query = {
mainQuery: `/**
applicationModeQuery: `/**
* @name Usage of APIs coming from external libraries
* @description A list of 3rd party APIs used in the codebase. Excludes test and generated code.
* @tags telemetry
@@ -29,6 +29,100 @@ where
supported = isSupported(api) and
usage = aUsage(api)
select usage, apiName, supported.toString(), "supported", api.jarContainer(), "library"
`,
frameworkModeQuery: `/**
* @name Public methods
* @description A list of APIs callable by consumers. Excludes test and generated code.
* @tags telemetry
* @kind problem
* @id java/telemetry/fetch-public-methods
*/
import java
private import semmle.code.java.dataflow.DataFlow
private import semmle.code.java.dataflow.ExternalFlow
private import semmle.code.java.dataflow.FlowSources
private import semmle.code.java.dataflow.FlowSummary
private import semmle.code.java.dataflow.internal.DataFlowPrivate
private import semmle.code.java.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
private import semmle.code.java.dataflow.internal.ModelExclusions
/** Holds if the given callable is not worth supporting. */
private predicate isUninteresting(Callable c) {
c.getDeclaringType() instanceof TestLibrary or
c.(Constructor).isParameterless()
}
class PublicMethod extends Callable {
PublicMethod() { this.isPublic() and not isUninteresting(this) }
/**
* Gets information about the method in the form expected by the MaD modeling framework.
*/
string getApiName() {
result =
this.getDeclaringType().getPackage() + "." + this.getDeclaringType().getSourceDeclaration() +
"#" + this.getName() + paramsString(this)
}
/** Gets a node that is an input to a call to this API. */
private DataFlow::Node getAnInput() {
exists(Call call | call.getCallee().getSourceDeclaration() = this |
result.asExpr().(Argument).getCall() = call or
result.(ArgumentNode).getCall().asCall() = call
)
}
/** Gets a node that is an output from a call to this API. */
private DataFlow::Node getAnOutput() {
exists(Call call | call.getCallee().getSourceDeclaration() = this |
result.asExpr() = call or
result.(DataFlow::PostUpdateNode).getPreUpdateNode().(ArgumentNode).getCall().asCall() = call
)
}
/** Holds if this API has a supported summary. */
pragma[nomagic]
predicate hasSummary() {
this = any(SummarizedCallable sc).asCallable() or
TaintTracking::localAdditionalTaintStep(this.getAnInput(), _)
}
pragma[nomagic]
predicate isSource() {
this.getAnOutput() instanceof RemoteFlowSource or sourceNode(this.getAnOutput(), _)
}
/** Holds if this API is a known sink. */
pragma[nomagic]
predicate isSink() { sinkNode(this.getAnInput(), _) }
/** Holds if this API is a known neutral. */
pragma[nomagic]
predicate isNeutral() { this = any(FlowSummaryImpl::Public::NeutralCallable nsc).asCallable() }
/**
* Holds if this API is supported by existing CodeQL libraries, that is, it is either a
* recognized source, sink or neutral or it has a flow summary.
*/
predicate isSupported() {
this.hasSummary() or this.isSource() or this.isSink() or this.isNeutral()
}
}
private boolean isSupported(PublicMethod publicMethod) {
publicMethod.isSupported() and result = true
or
not publicMethod.isSupported() and result = false
}
from PublicMethod publicMethod, string apiName, boolean supported
where
apiName = publicMethod.getApiName() and
publicMethod.getCompilationUnit().isSourceFile() and
supported = isSupported(publicMethod)
select publicMethod, apiName, supported.toString(), "supported",
publicMethod.getCompilationUnit().getParentContainer().getBaseName(), "library"
`,
dependencies: {
"ExternalApi.qll": `/** Provides classes and predicates related to handling APIs from external libraries. */

19
extensions/ql-vscode/src/data-extensions-editor/queries/query.ts
View File

@@ -1,16 +1,29 @@
export type Query = {
/**
* The main query.
* The application query.
*
* It should select all usages of external APIs, and return the following result pattern:
* - usage: the usage of the external API. This is an entity.
* - apiName: the name of the external API. This is a string.
* - supported: whether the external API is supported by the extension. This should be a string representation of a boolean to satify the result pattern for a problem query.
* - supported: whether the external API is modeled. This should be a string representation of a boolean to satify the result pattern for a problem query.
* - "supported": a string literal. This is required to make the query a valid problem query.
* - libraryName: the name of the library that contains the external API. This is a string and usually the basename of a file.
* - "library": a string literal. This is required to make the query a valid problem query.
*/
mainQuery: string;
applicationModeQuery: string;
/**
* The framework query.
*
* It should select all methods that are callable by applications, which is usually all public methods (and constructors).
* The result pattern should be as follows:
* - method: the method that is callable by applications. This is an entity.
* - apiName: the name of the external API. This is a string.
* - supported: whether this method is modeled. This should be a string representation of a boolean to satify the result pattern for a problem query.
* - "supported": a string literal. This is required to make the query a valid problem query.
* - libraryName: an arbitrary string. This is required to make it match the structure of the application query.
* - "library": a string literal. This is required to make the query a valid problem query.
*/
frameworkModeQuery: string;
dependencies?: {
[filename: string]: string;
};

4
extensions/ql-vscode/src/data-extensions-editor/shared/mode.ts Normal file
View File

@@ -0,0 +1,4 @@
export enum Mode {
Application = "application",
Framework = "framework",
}

3
extensions/ql-vscode/src/data-extensions-editor/shared/view-state.ts
View File

@@ -1,6 +1,9 @@
import { ExtensionPack } from "./extension-pack";
import { Mode } from "./mode";
export interface DataExtensionEditorViewState {
extensionPack: ExtensionPack;
enableFrameworkMode: boolean;
showLlmButton: boolean;
mode: Mode;
}

30
extensions/ql-vscode/src/data-extensions-editor/yaml.ts
View File

@@ -10,6 +10,7 @@ import {
} from "./predicates";
import * as dataSchemaJson from "./data-schema.json";
import { sanitizeExtensionPackName } from "./extension-pack-name";
const ajv = new Ajv({ allErrors: true });
const dataSchemaValidate = ajv.compile(dataSchemaJson);
@@ -79,7 +80,7 @@ export function createDataExtensionYaml(
${extensions.join("\n")}`;
}
export function createDataExtensionYamlsPerLibrary(
export function createDataExtensionYamlsForApplicationMode(
language: string,
externalApiUsages: ExternalApiUsage[],
modeledMethods: Record<string, ModeledMethod>,
@@ -116,6 +117,33 @@ export function createDataExtensionYamlsPerLibrary(
return result;
}
export function createDataExtensionYamlsForFrameworkMode(
databaseName: string,
language: string,
externalApiUsages: ExternalApiUsage[],
modeledMethods: Record<string, ModeledMethod>,
prefix = "models/",
suffix = ".model",
): Record<string, string> {
const parts = databaseName.split("/");
const libraryName = parts
.slice(1)
.map((part) => sanitizeExtensionPackName(part))
.join("-");
const methods = externalApiUsages.map((externalApiUsage) => ({
externalApiUsage,
modeledMethod: modeledMethods[externalApiUsage.signature],
}));
return {
[`${prefix}${libraryName}${suffix}.yml`]: createDataExtensionYaml(
language,
methods,
),
};
}
// From the semver package using
// const { re, t } = require("semver/internal/re");
// console.log(re[t.LOOSE]);

3
extensions/ql-vscode/src/stories/data-extensions-editor/DataExtensionsEditor.stories.tsx
View File

@@ -2,6 +2,7 @@ import * as React from "react";
import { ComponentMeta, ComponentStory } from "@storybook/react";
import { Mode } from "../../data-extensions-editor/shared/mode";
import { DataExtensionsEditor as DataExtensionsEditorComponent } from "../../view/data-extensions-editor/DataExtensionsEditor";
export default {
@@ -25,7 +26,9 @@ DataExtensionsEditor.args = {
extensionTargets: {},
dataExtensions: [],
},
enableFrameworkMode: true,
showLlmButton: true,
mode: Mode.Application,
},
initialExternalApiUsages: [
{

35
extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx
View File

@@ -16,6 +16,7 @@ import { ViewTitle } from "../common";
import { DataExtensionEditorViewState } from "../../data-extensions-editor/shared/view-state";
import { ModeledMethodsList } from "./ModeledMethodsList";
import { percentFormatter } from "./formatters";
import { Mode } from "../../data-extensions-editor/shared/mode";
const DataExtensionsEditorContainer = styled.div`
margin-top: 1rem;
@@ -166,6 +167,16 @@ export function DataExtensionsEditor({
});
}, []);
const onSwitchModeClick = useCallback(() => {
const newMode =
viewState?.mode === Mode.Framework ? Mode.Application : Mode.Framework;
vscode.postMessage({
t: "switchMode",
mode: newMode,
});
}, [viewState?.mode]);
return (
<DataExtensionsEditorContainer>
{progress.maxStep > 0 && (
@@ -193,13 +204,34 @@ export function DataExtensionsEditor({
<div>
{percentFormatter.format(unModeledPercentage / 100)} unmodeled
</div>
{viewState?.enableFrameworkMode && (
<>
<div>
Mode:{" "}
{viewState?.mode === Mode.Framework
? "Framework"
: "Application"}
</div>
<div>
<LinkIconButton onClick={onSwitchModeClick}>
<span
slot="start"
className="codicon codicon-library"
></span>
Switch mode
</LinkIconButton>
</div>
</>
)}
</DetailsContainer>
<EditorContainer>
<ButtonsContainer>
<VSCodeButton onClick={onApplyClick}>Apply</VSCodeButton>
<VSCodeButton onClick={onGenerateClick}>
Download and generate
{viewState?.mode === Mode.Framework
? "Generate"
: "Download and generate"}
</VSCodeButton>
{viewState?.showLlmButton && (
<>
@@ -212,6 +244,7 @@ export function DataExtensionsEditor({
<ModeledMethodsList
externalApiUsages={externalApiUsages}
modeledMethods={modeledMethods}
mode={viewState?.mode ?? Mode.Application}
onChange={onChange}
/>
</EditorContainer>

10
extensions/ql-vscode/src/view/data-extensions-editor/LibraryRow.tsx
View File

@@ -8,6 +8,7 @@ import { ModeledMethodDataGrid } from "./ModeledMethodDataGrid";
import { calculateModeledPercentage } from "./modeled";
import { decimalFormatter, percentFormatter } from "./formatters";
import { Codicon } from "../common";
import { Mode } from "../../data-extensions-editor/shared/mode";
const LibraryContainer = styled.div`
margin-bottom: 1rem;
@@ -38,9 +39,10 @@ const StatusContainer = styled.div`
`;
type Props = {
libraryName: string;
title: string;
externalApiUsages: ExternalApiUsage[];
modeledMethods: Record<string, ModeledMethod>;
mode: Mode;
onChange: (
externalApiUsage: ExternalApiUsage,
modeledMethod: ModeledMethod,
@@ -48,9 +50,10 @@ type Props = {
};
export const LibraryRow = ({
libraryName,
title,
externalApiUsages,
modeledMethods,
mode,
onChange,
}: Props) => {
const modeledPercentage = useMemo(() => {
@@ -75,7 +78,7 @@ export const LibraryRow = ({
) : (
<Codicon name="chevron-right" label="Expand" />
)}
{libraryName}
{title}
{isExpanded ? null : (
<>
{" "}
@@ -116,6 +119,7 @@ export const LibraryRow = ({
<ModeledMethodDataGrid
externalApiUsages={externalApiUsages}
modeledMethods={modeledMethods}
mode={mode}
onChange={onChange}
/>
</>

5
extensions/ql-vscode/src/view/data-extensions-editor/MethodRow.tsx
View File

@@ -17,6 +17,7 @@ import {
} from "../../data-extensions-editor/modeled-method";
import { KindInput } from "./KindInput";
import { extensiblePredicateDefinitions } from "../../data-extensions-editor/predicates";
import { Mode } from "../../data-extensions-editor/shared/mode";
const Dropdown = styled(VSCodeDropdown)`
width: 100%;
@@ -47,6 +48,7 @@ const UsagesButton = styled.button`
type Props = {
externalApiUsage: ExternalApiUsage;
modeledMethod: ModeledMethod | undefined;
mode: Mode;
onChange: (
externalApiUsage: ExternalApiUsage,
modeledMethod: ModeledMethod,
@@ -56,6 +58,7 @@ type Props = {
export const MethodRow = ({
externalApiUsage,
modeledMethod,
mode,
onChange,
}: Props) => {
const argumentsList = useMemo(() => {
@@ -165,11 +168,13 @@ export const MethodRow = ({
{externalApiUsage.methodParameters}
</SupportSpan>
</VSCodeDataGridCell>
{mode === Mode.Application && (
<VSCodeDataGridCell gridColumn={3}>
<UsagesButton onClick={jumpToUsage}>
{externalApiUsage.usages.length}
</UsagesButton>
</VSCodeDataGridCell>
)}
<VSCodeDataGridCell gridColumn={4}>
{(!externalApiUsage.supported ||
(modeledMethod && modeledMethod?.type !== "none")) && (

6
extensions/ql-vscode/src/view/data-extensions-editor/ModeledMethodDataGrid.tsx
View File

@@ -8,10 +8,12 @@ import { MethodRow } from "./MethodRow";
import { ExternalApiUsage } from "../../data-extensions-editor/external-api-usage";
import { ModeledMethod } from "../../data-extensions-editor/modeled-method";
import { useMemo } from "react";
import { Mode } from "../../data-extensions-editor/shared/mode";
type Props = {
externalApiUsages: ExternalApiUsage[];
modeledMethods: Record<string, ModeledMethod>;
mode: Mode;
onChange: (
externalApiUsage: ExternalApiUsage,
modeledMethod: ModeledMethod,
@@ -21,6 +23,7 @@ type Props = {
export const ModeledMethodDataGrid = ({
externalApiUsages,
modeledMethods,
mode,
onChange,
}: Props) => {
const sortedExternalApiUsages = useMemo(() => {
@@ -48,9 +51,11 @@ export const ModeledMethodDataGrid = ({
<VSCodeDataGridCell cellType="columnheader" gridColumn={2}>
Method
</VSCodeDataGridCell>
{mode === Mode.Application && (
<VSCodeDataGridCell cellType="columnheader" gridColumn={3}>
Usages
</VSCodeDataGridCell>
)}
<VSCodeDataGridCell cellType="columnheader" gridColumn={4}>
Model type
</VSCodeDataGridCell>
@@ -69,6 +74,7 @@ export const ModeledMethodDataGrid = ({
key={externalApiUsage.signature}
externalApiUsage={externalApiUsage}
modeledMethod={modeledMethods[externalApiUsage.signature]}
mode={mode}
onChange={onChange}
/>
))}

46
extensions/ql-vscode/src/view/data-extensions-editor/ModeledMethodsList.tsx
View File

@@ -4,10 +4,12 @@ import { ExternalApiUsage } from "../../data-extensions-editor/external-api-usag
import { ModeledMethod } from "../../data-extensions-editor/modeled-method";
import { calculateModeledPercentage } from "./modeled";
import { LibraryRow } from "./LibraryRow";
import { Mode } from "../../data-extensions-editor/shared/mode";
type Props = {
externalApiUsages: ExternalApiUsage[];
modeledMethods: Record<string, ModeledMethod>;
mode: Mode;
onChange: (
externalApiUsage: ExternalApiUsage,
modeledMethod: ModeledMethod,
@@ -17,27 +19,30 @@ type Props = {
export const ModeledMethodsList = ({
externalApiUsages,
modeledMethods,
mode,
onChange,
}: Props) => {
const groupedByLibrary = useMemo(() => {
const grouped = useMemo(() => {
const groupedByLibrary: Record<string, ExternalApiUsage[]> = {};
for (const externalApiUsage of externalApiUsages) {
groupedByLibrary[externalApiUsage.library] ??= [];
groupedByLibrary[externalApiUsage.library].push(externalApiUsage);
// Group by package if using framework mode
const key =
mode === Mode.Framework
? externalApiUsage.packageName
: externalApiUsage.library;
groupedByLibrary[key] ??= [];
groupedByLibrary[key].push(externalApiUsage);
}
return groupedByLibrary;
}, [externalApiUsages]);
}, [externalApiUsages, mode]);
const sortedLibraryNames = useMemo(() => {
return Object.keys(groupedByLibrary).sort((a, b) => {
const supportedPercentageA = calculateModeledPercentage(
groupedByLibrary[a],
);
const supportedPercentageB = calculateModeledPercentage(
groupedByLibrary[b],
);
const sortedGroupNames = useMemo(() => {
return Object.keys(grouped).sort((a, b) => {
const supportedPercentageA = calculateModeledPercentage(grouped[a]);
const supportedPercentageB = calculateModeledPercentage(grouped[b]);
// Sort first by supported percentage ascending
if (supportedPercentageA > supportedPercentageB) {
@@ -47,19 +52,19 @@ export const ModeledMethodsList = ({
return -1;
}
const numberOfUsagesA = groupedByLibrary[a].reduce(
const numberOfUsagesA = grouped[a].reduce(
(acc, curr) => acc + curr.usages.length,
0,
);
const numberOfUsagesB = groupedByLibrary[b].reduce(
const numberOfUsagesB = grouped[b].reduce(
(acc, curr) => acc + curr.usages.length,
0,
);
// If the number of usages is equal, sort by number of methods descending
if (numberOfUsagesA === numberOfUsagesB) {
const numberOfMethodsA = groupedByLibrary[a].length;
const numberOfMethodsB = groupedByLibrary[b].length;
const numberOfMethodsA = grouped[a].length;
const numberOfMethodsB = grouped[b].length;
// If the number of methods is equal, sort by library name ascending
if (numberOfMethodsA === numberOfMethodsB) {
@@ -72,16 +77,17 @@ export const ModeledMethodsList = ({
// Then sort by number of usages descending
return numberOfUsagesB - numberOfUsagesA;
});
}, [groupedByLibrary]);
}, [grouped]);
return (
<>
{sortedLibraryNames.map((libraryName) => (
{sortedGroupNames.map((libraryName) => (
<LibraryRow
key={libraryName}
libraryName={libraryName}
externalApiUsages={groupedByLibrary[libraryName]}
title={libraryName}
externalApiUsages={grouped[libraryName]}
modeledMethods={modeledMethods}
mode={mode}
onChange={onChange}
/>
))}

143
extensions/ql-vscode/test/unit-tests/data-extensions-editor/yaml.test.ts
View File

@@ -1,6 +1,7 @@
import {
createDataExtensionYaml,
createDataExtensionYamlsPerLibrary,
createDataExtensionYamlsForApplicationMode,
createDataExtensionYamlsForFrameworkMode,
createFilenameForLibrary,
loadDataExtensionYaml,
} from "../../../src/data-extensions-editor/yaml";
@@ -134,9 +135,9 @@ describe("createDataExtensionYaml", () => {
});
});
describe("createDataExtensionYamlsPerLibrary", () => {
describe("createDataExtensionYamlsForApplicationMode", () => {
it("creates the correct YAML files", () => {
const yaml = createDataExtensionYamlsPerLibrary(
const yaml = createDataExtensionYamlsForApplicationMode(
"java",
[
{
@@ -341,6 +342,142 @@ describe("createDataExtensionYamlsPerLibrary", () => {
});
});
describe("createDataExtensionYamlsForFrameworkMode", () => {
it("creates the correct YAML files", () => {
const yaml = createDataExtensionYamlsForFrameworkMode(
"github/sql2o",
"java",
[
{
library: "sql2o",
signature: "org.sql2o.Connection#createQuery(String)",
packageName: "org.sql2o",
typeName: "Connection",
methodName: "createQuery",
methodParameters: "(String)",
supported: true,
usages: [
{
label: "createQuery(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 15,
startColumn: 13,
endLine: 15,
endColumn: 56,
},
},
{
label: "createQuery(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 26,
startColumn: 13,
endLine: 26,
endColumn: 39,
},
},
],
},
{
library: "sql2o",
signature: "org.sql2o.Query#executeScalar(Class)",
packageName: "org.sql2o",
typeName: "Query",
methodName: "executeScalar",
methodParameters: "(Class)",
supported: true,
usages: [
{
label: "executeScalar(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 15,
startColumn: 13,
endLine: 15,
endColumn: 85,
},
},
{
label: "executeScalar(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 26,
startColumn: 13,
endLine: 26,
endColumn: 68,
},
},
],
},
{
library: "sql2o",
signature: "org.sql2o.Sql2o#Sql2o(String,String,String)",
packageName: "org.sql2o",
typeName: "Sql2o",
methodName: "Sql2o",
methodParameters: "(String,String,String)",
supported: false,
usages: [
{
label: "new Sql2o(...)",
url: {
uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
startLine: 10,
startColumn: 33,
endLine: 10,
endColumn: 88,
},
},
],
},
],
{
"org.sql2o.Connection#createQuery(String)": {
type: "sink",
input: "Argument[0]",
output: "",
kind: "sql",
provenance: "df-generated",
},
"org.sql2o.Sql2o#Sql2o(String,String,String)": {
type: "sink",
input: "Argument[0]",
output: "",
kind: "jndi",
provenance: "manual",
},
},
);
expect(yaml).toEqual({
"models/sql2o.model.yml": `extensions:
- addsTo:
pack: codeql/java-all
extensible: sourceModel
data: []
- addsTo:
pack: codeql/java-all
extensible: sinkModel
data:
- ["org.sql2o","Connection",true,"createQuery","(String)","","Argument[0]","sql","df-generated"]
- ["org.sql2o","Sql2o",true,"Sql2o","(String,String,String)","","Argument[0]","jndi","manual"]
- addsTo:
pack: codeql/java-all
extensible: summaryModel
data: []
- addsTo:
pack: codeql/java-all
extensible: neutralModel
data: []
`,
});
});
});
describe("loadDataExtensionYaml", () => {
it("loads the YAML file", () => {
const data = loadDataExtensionYaml({

41
extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts
View File

@@ -14,6 +14,8 @@ import { fetchExternalApiQueries } from "../../../../src/data-extensions-editor/
import * as log from "../../../../src/common/logging/notifications";
import { RedactableError } from "../../../../src/common/errors";
import { showAndLogExceptionWithTelemetry } from "../../../../src/common/logging";
import { QueryLanguage } from "../../../../src/common/query-language";
import { Query } from "../../../../src/data-extensions-editor/queries/query";
function createMockUri(path = "/a/b/c/foo"): Uri {
return {
@@ -29,11 +31,31 @@ function createMockUri(path = "/a/b/c/foo"): Uri {
}
describe("runQuery", () => {
it("runs all queries", async () => {
const cases = Object.keys(fetchExternalApiQueries).flatMap((lang) => {
const query = fetchExternalApiQueries[lang as QueryLanguage];
if (!query) {
return [];
}
const keys = new Set(Object.keys(query));
keys.delete("dependencies");
return Array.from(keys).map((name) => ({
language: lang as QueryLanguage,
queryName: name as keyof Omit<Query, "dependencies">,
}));
});
test.each(cases)(
"should run $queryName for $language",
async ({ language, queryName }) => {
const logPath = (await file()).path;
// Test all queries
for (const [lang, query] of Object.entries(fetchExternalApiQueries)) {
const query = fetchExternalApiQueries[language];
if (!query) {
throw new Error(`No query found for language ${language}`);
}
const options = {
cliServer: {
resolveQlpacks: jest.fn().mockResolvedValue({
@@ -58,7 +80,7 @@ describe("runQuery", () => {
name: "foo",
datasetUri: createMockUri(),
},
language: lang,
language,
},
queryStorageDir: "/tmp/queries",
progress: jest.fn(),
@@ -67,7 +89,8 @@ describe("runQuery", () => {
onCancellationRequested: jest.fn(),
},
};
const result = await runQuery(options);
const result = await runQuery(queryName, options);
expect(result?.resultType).toEqual(QueryResultType.SUCCESS);
@@ -106,13 +129,13 @@ describe("runQuery", () => {
name: "codeql/external-api-usage",
version: "0.0.0",
dependencies: {
[`codeql/${lang}-all`]: "*",
[`codeql/${language}-all`]: "*",
},
});
expect(
await readFile(join(queryDirectory, "FetchExternalApis.ql"), "utf8"),
).toEqual(query.mainQuery);
).toEqual(query[queryName]);
for (const [filename, contents] of Object.entries(
query.dependencies ?? {},
@@ -121,8 +144,8 @@ describe("runQuery", () => {
contents,
);
}
}
});
},
);
});
describe("readQueryResults", () => {