Graph viewer support

This commit is contained in:
Tom Hvitved
2021-07-14 12:34:39 +02:00
committed by Andrew Eisenberg
parent ddca0bb851
commit 580832ea7b
15 changed files with 1762 additions and 801 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -48,6 +48,7 @@
"onCommand:codeQLDatabases.chooseDatabaseLgtm",
"onCommand:codeQL.setCurrentDatabase",
"onCommand:codeQL.viewAst",
"onCommand:codeQL.viewCfg",
"onCommand:codeQL.openReferencedFile",
"onCommand:codeQL.previewQueryHelp",
"onCommand:codeQL.chooseDatabaseFolder",
@@ -368,6 +369,10 @@
"command": "codeQL.viewAst",
"title": "CodeQL: View AST"
},
{
"command": "codeQL.viewCfg",
"title": "CodeQL: View CFG"
},
{
"command": "codeQL.upgradeCurrentDatabase",
"title": "CodeQL: Upgrade Current Database"
@@ -737,6 +742,11 @@
"group": "9_qlCommands",
"when": "resourceScheme == codeql-zip-archive && !explorerResourceIsFolder && !listMultiSelection"
},
{
"command": "codeQL.viewCfg",
"group": "9_qlCommands",
"when": "resourceScheme == codeql-zip-archive"
},
{
"command": "codeQL.runQueries",
"group": "9_qlCommands",
@@ -798,6 +808,10 @@
"command": "codeQL.viewAst",
"when": "resourceScheme == codeql-zip-archive"
},
{
"command": "codeQL.viewCfg",
"when": "resourceScheme == codeql-zip-archive"
},
{
"command": "codeQLDatabases.setCurrentDatabase",
"when": "false"
@@ -1017,6 +1031,8 @@
"@primer/react": "^34.3.0",
"child-process-promise": "^2.2.1",
"classnames": "~2.2.6",
"d3": "^6.3.1",
"d3-graphviz": "^2.6.1",
"fs-extra": "^9.0.1",
"glob-promise": "^3.4.0",
"js-yaml": "^3.14.0",
@@ -1048,6 +1064,8 @@
"@types/child-process-promise": "^2.2.1",
"@types/classnames": "~2.2.9",
"@types/del": "^4.0.0",
"@types/d3": "^6.2.0",
"@types/d3-graphviz": "^2.6.6",
"@types/fs-extra": "^9.0.6",
"@types/glob": "^7.1.1",
"@types/google-protobuf": "^3.2.7",

View File

@@ -8,4 +8,4 @@
* succeeds.
*/
declare type Blob = string;
//declare type Blob = string; // TODO: Check this

View File

@@ -1,5 +1,6 @@
import * as cpp from 'child-process-promise';
import * as child_process from 'child_process';
import * as fs from 'fs-extra';
import * as path from 'path';
import * as sarif from 'sarif';
import { SemVer } from 'semver';
@@ -715,6 +716,26 @@ export class CodeQLCliServer implements Disposable {
return await sarifParser(interpretedResultsPath);
}
async readDotFiles(dir: string): Promise<string[]> {
return Promise.all((await fs.readdir(dir))
.filter(name => path.extname(name).toLowerCase() === '.dot')
.map(file => fs.readFile(path.join(dir, file), 'utf8'))
);
}
async interpretBqrsGraph(metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<string[]> {
const additionalArgs = sourceInfo ? ['--dot-location-url-format', 'file://' + sourceInfo.sourceLocationPrefix + '{path}:{start:line}:{start:column}:{end:line}:{end:column}'] : [];
await this.runInterpretCommand('dot', additionalArgs, metadata, resultsPath, interpretedResultsPath, sourceInfo);
try {
const dot = await this.readDotFiles(interpretedResultsPath);
return dot;
} catch (err) {
throw new Error(`Reading output of interpretation failed: ${err.stderr || err}`);
}
}
async generateResultsCsv(metadata: QueryMetadata, resultsPath: string, csvPath: string, sourceInfo?: SourceInfo): Promise<void> {
await this.runInterpretCommand(CSV_FORMAT, [], metadata, resultsPath, csvPath, sourceInfo);
}

View File

@@ -2,6 +2,7 @@ export enum KeyType {
DefinitionQuery = 'DefinitionQuery',
ReferenceQuery = 'ReferenceQuery',
PrintAstQuery = 'PrintAstQuery',
PrintCfgQuery = 'PrintCfgQuery',
}
export function tagOfKeyType(keyType: KeyType): string {
@@ -12,6 +13,8 @@ export function tagOfKeyType(keyType: KeyType): string {
return 'ide-contextual-queries/local-references';
case KeyType.PrintAstQuery:
return 'ide-contextual-queries/print-ast';
case KeyType.PrintCfgQuery:
return 'ide-contextual-queries/print-cfg';
}
}
@@ -23,6 +26,8 @@ export function nameOfKeyType(keyType: KeyType): string {
return 'references';
case KeyType.PrintAstQuery:
return 'print AST';
case KeyType.PrintCfgQuery:
return 'print CFG';
}
}
@@ -32,6 +37,7 @@ export function kindOfKeyType(keyType: KeyType): string {
case KeyType.ReferenceQuery:
return 'definitions';
case KeyType.PrintAstQuery:
case KeyType.PrintCfgQuery:
return 'graph';
}
}

View File

@@ -224,3 +224,62 @@ export class TemplatePrintAstProvider {
};
}
}
export class TemplatePrintCfgProvider {
private cache: CachedOperation<[Uri, messages.TemplateDefinitions] | undefined>;
constructor(
private cli: CodeQLCliServer,
private dbm: DatabaseManager
) {
this.cache = new CachedOperation<[Uri, messages.TemplateDefinitions] | undefined>(this.getCfgUri.bind(this));
}
async provideCfgUri(document?: TextDocument): Promise<[Uri, messages.TemplateDefinitions] | undefined> {
if (!document) {
return;
}
return await this.cache.get(document.uri.toString());
}
private async getCfgUri(uriString: string): Promise<[Uri, messages.TemplateDefinitions]> {
const uri = Uri.parse(uriString, true);
if (uri.scheme !== zipArchiveScheme) {
throw new Error('CFG Viewing is only available for databases with zipped source archives.');
}
const zippedArchive = decodeSourceArchiveUri(uri);
const sourceArchiveUri = encodeArchiveBasePath(zippedArchive.sourceArchiveZipPath);
const db = this.dbm.findDatabaseItemBySourceArchive(sourceArchiveUri);
if (!db) {
throw new Error('Can\'t infer database from the provided source.');
}
const qlpack = await qlpackOfDatabase(this.cli, db);
if (!qlpack) {
throw new Error('Can\'t infer qlpack from database source archive');
}
const queries = await resolveQueries(this.cli, qlpack, KeyType.PrintCfgQuery);
if (queries.length > 1) {
throw new Error('Found multiple Print CFG queries. Can\'t continue');
}
if (queries.length === 0) {
throw new Error('Did not find any Print CFG queries. Can\'t continue');
}
const queryUri = Uri.file(queries[0]);
const templates: messages.TemplateDefinitions = {
[TEMPLATE_NAME]: {
values: {
tuples: [[{
stringValue: zippedArchive.pathWithinSourceArchive
}]]
}
}
};
return [queryUri, templates];
}
}

View File

@@ -41,7 +41,8 @@ import { DatabaseUI } from './databases-ui';
import {
TemplateQueryDefinitionProvider,
TemplateQueryReferenceProvider,
TemplatePrintAstProvider
TemplatePrintAstProvider,
TemplatePrintCfgProvider
} from './contextual/templateProvider';
import {
DEFAULT_DISTRIBUTION_VERSION_RANGE,
@@ -981,6 +982,26 @@ async function activateWithInstalledDistribution(
})
);
ctx.subscriptions.push(
commandRunnerWithProgress(
'codeQL.viewCfg',
async (
progress: ProgressCallback,
token: CancellationToken
) => {
const res = await new TemplatePrintCfgProvider(cliServer, dbm)
.provideCfgUri(window.activeTextEditor?.document);
if (res) {
await compileAndRunQuery(false, res[0], progress, token, undefined);
}
},
{
title: 'Calculate CFG',
cancellable: true
}
)
);
void logger.log('Starting language server.');
ctx.subscriptions.push(client.start());

View File

@@ -27,12 +27,13 @@ import {
InterpretedResultsSortState,
SortDirection,
ALERTS_TABLE_NAME,
GRAPH_TABLE_NAME,
RawResultsSortState,
} from './pure/interface-types';
import { Logger } from './logging';
import * as messages from './pure/messages';
import { commandRunner } from './commandRunner';
import { CompletedQueryInfo, interpretResultsSarif } from './query-results';
import { CompletedQueryInfo, interpretResultsSarif, interpretGraphResults } from './query-results';
import { QueryEvaluationInfo, tmpDir } from './run-queries';
import { parseSarifLocation, parseSarifPlainTextMessage } from './pure/sarif-utils';
import {
@@ -88,12 +89,33 @@ function sortInterpretedResults(
}
}
function numPagesOfResultSet(resultSet: RawResultSet): number {
return Math.ceil(resultSet.schema.rows / PAGE_SIZE.getValue<number>());
function interpretedPageSize(interpretation: Interpretation | undefined): number {
if (interpretation && interpretation.data.t == 'GraphInterpretationData')
return 1;
return PAGE_SIZE.getValue<number>();
}
function numPagesOfResultSet(resultSet: RawResultSet, interpretation?: Interpretation): number {
const pageSize = interpretedPageSize(interpretation);
const n = interpretation && interpretation.data.t == 'GraphInterpretationData'
? interpretation.data.dot.length
: resultSet.schema.rows;
return Math.ceil(n / pageSize);
}
function numInterpretedPages(interpretation: Interpretation | undefined): number {
return Math.ceil((interpretation?.data.runs[0].results?.length || 0) / PAGE_SIZE.getValue<number>());
if (!interpretation)
return 0;
const pageSize = interpretedPageSize(interpretation);
const n = interpretation.data.t == 'GraphInterpretationData'
? interpretation.data.dot.length
: interpretation.data.runs[0].results?.length || 0;
return Math.ceil(n / pageSize);
}
export class InterfaceManager extends DisposableObject {
@@ -305,7 +327,7 @@ export class InterfaceManager extends DisposableObject {
await this.changeInterpretedSortState(msg.sortState);
break;
case 'changePage':
if (msg.selectedTable === ALERTS_TABLE_NAME) {
if (msg.selectedTable === ALERTS_TABLE_NAME || msg.selectedTable === GRAPH_TABLE_NAME) {
await this.showPageOfInterpretedResults(msg.pageNumber);
}
else {
@@ -325,8 +347,8 @@ export class InterfaceManager extends DisposableObject {
break;
default:
assertNever(msg);
}
} catch (e) {
}
} catch (e) {
void showAndLogErrorMessage(e.message, {
fullMessage: e.stack
});
@@ -438,7 +460,7 @@ export class InterfaceManager extends DisposableObject {
const parsedResultSets: ParsedResultSets = {
pageNumber: 0,
pageSize,
numPages: numPagesOfResultSet(resultSet),
numPages: numPagesOfResultSet(resultSet, this._interpretation),
numInterpretedPages: numInterpretedPages(this._interpretation),
resultSet: { ...resultSet, t: 'RawResultSet' },
selectedTable: undefined,
@@ -488,7 +510,7 @@ export class InterfaceManager extends DisposableObject {
metadata: this._displayedQuery.completedQuery.query.metadata,
pageNumber,
resultSetNames,
pageSize: PAGE_SIZE.getValue(),
pageSize: interpretedPageSize(this._interpretation),
numPages: numInterpretedPages(this._interpretation),
queryName: this._displayedQuery.label,
queryPath: this._displayedQuery.initialInfo.queryPath
@@ -586,30 +608,50 @@ export class InterfaceManager extends DisposableObject {
sourceInfo: cli.SourceInfo | undefined,
sourceLocationPrefix: string,
sortState: InterpretedResultsSortState | undefined
): Promise<Interpretation | undefined> {
): Promise<Interpretation | undefined> {
if (!resultsPaths) {
void this.logger.log('No results path. Cannot display interpreted results.');
return undefined;
}
let data;
let numTotalResults;
if (metadata?.kind === 'graph')
{
data = await interpretGraphResults(
this.cliServer,
metadata,
resultsPaths,
sourceInfo
);
numTotalResults = data.dot.length;
}
else
{
const sarif = await interpretResultsSarif(
this.cliServer,
metadata,
resultsPaths,
sourceInfo
);
const sarif = await interpretResultsSarif(
this.cliServer,
metadata,
resultsPaths,
sourceInfo
);
sarif.runs.forEach(run => {
if (run.results !== undefined) {
sortInterpretedResults(run.results, sortState);
}
});
sarif.runs.forEach(run => {
if (run.results !== undefined) {
sortInterpretedResults(run.results, sortState);
}
});
sarif.sortState = sortState;
data = sarif;
const numTotalResults = sarif.runs[0]?.results?.length || 0;
numTotalResults = (() => {
if (sarif.runs.length === 0) return 0;
if (sarif.runs[0].results === undefined) return 0;
return sarif.runs[0].results.length;
})();
}
sarif.sortState = sortState;
const interpretation: Interpretation = {
data: sarif,
data,
sourceLocationPrefix,
numTruncatedResults: 0,
numTotalResults

View File

@@ -10,6 +10,7 @@ import { RawResultSet, ResultRow, ResultSetSchema, Column, ResolvableLocationVal
export const SELECT_TABLE_NAME = '#select';
export const ALERTS_TABLE_NAME = 'alerts';
export const GRAPH_TABLE_NAME = 'graph';
export type RawTableResultSet = { t: 'RawResultSet' } & RawResultSet;
export type InterpretedResultSet<T> = {
@@ -56,8 +57,12 @@ export type SarifInterpretationData = {
sortState?: InterpretedResultsSortState;
} & sarif.Log;
// Add more interpretation data kinds when needed (e.g., graph data)
export type InterpretationData = SarifInterpretationData;
export type GraphInterpretationData = {
t: 'GraphInterpretationData';
dot: string[];
};
export type InterpretationData = SarifInterpretationData | GraphInterpretationData;
export interface InterpretationT<T> {
sourceLocationPrefix: string;
@@ -367,6 +372,7 @@ export function getDefaultResultSetName(
// Choose first available result set from the array
return [
ALERTS_TABLE_NAME,
GRAPH_TABLE_NAME,
SELECT_TABLE_NAME,
resultSetNames[0]
].filter((resultSetName) => resultSetNames.includes(resultSetName))[0];

View File

@@ -11,7 +11,8 @@ import {
QueryMetadata,
InterpretedResultsSortState,
ResultsPaths,
SarifInterpretationData
SarifInterpretationData,
GraphInterpretationData
} from './pure/interface-types';
import { QueryHistoryConfig } from './config';
import { DatabaseInfo } from './pure/interface-types';
@@ -179,7 +180,6 @@ export function ensureMetadataIsComplete(metadata: QueryMetadata | undefined) {
return metadata;
}
/**
* Used in Interface and Compare-Interface for queries that we know have been complated.
*/
@@ -380,3 +380,22 @@ export class FullQueryInfo {
});
}
}
/**
* Call cli command to interpret graph results.
*/
export async function interpretGraphResults(
server: cli.CodeQLCliServer,
metadata: QueryMetadata | undefined,
resultsPaths: ResultsPaths,
sourceInfo?: cli.SourceInfo
): Promise<GraphInterpretationData> {
const { resultsPath, interpretedResultsPath } = resultsPaths;
if (await fs.pathExists(interpretedResultsPath)) {
const dot = await server.readDotFiles(interpretedResultsPath);
return { dot, t: 'GraphInterpretationData' };
}
const dot = await server.interpretBqrsGraph(ensureMetadataIsComplete(metadata), resultsPath, interpretedResultsPath, sourceInfo);
return { dot, t: 'GraphInterpretationData' };
}

View File

@@ -0,0 +1,74 @@
import * as React from 'react';
import * as d3 from 'd3';
import { ResultTableProps } from './result-table-utils';
import { InterpretedResultSet, GraphInterpretationData } from '../pure/interface-types';
import { graphviz } from 'd3-graphviz';
import { jumpToLocation } from './result-table-utils';
import { tryGetLocationFromString } from '../pure/bqrs-utils';
export type GraphProps = ResultTableProps & { resultSet: InterpretedResultSet<GraphInterpretationData> };
const className = 'vscode-codeql__result-tables-graph';
export class Graph extends React.Component<GraphProps> {
constructor(props: GraphProps) {
super(props);
}
public render = (): JSX.Element => {
return <div id={className} className={className} />;
};
public componentDidMount = () => {
this.renderGraph();
};
public componentDidUpdate = () => {
this.renderGraph();
};
private renderGraph = () => {
const { databaseUri, resultSet } = this.props;
const options = {
fit: true,
fade: false,
growEnteringEdges: false,
zoom: true,
};
const element = document.querySelector(`.${className}`);
const color = element ? getComputedStyle(element).color : 'black';
const backgroundColor = element ? getComputedStyle(element).backgroundColor : 'transparent';
const borderColor = element ? getComputedStyle(element).borderColor : 'black';
let firstPolygon = true;
graphviz(`#${className}`)
.options(options)
.attributer(function(d) {
if (d.tag == 'a') {
const url = d.attributes['xlink:href'] || d.attributes['href'];
const loc = tryGetLocationFromString(url);
if (loc !== undefined) {
d.attributes['xlink:href'] = '#';
d.attributes['href'] = '#';
loc.uri = 'file://' + loc.uri;
d3.select(this).on('click', function(e) { jumpToLocation(loc, databaseUri); });
}
}
if ('fill' in d.attributes) {
d.attributes.fill = d.tag == 'text' ? color : backgroundColor;
}
if ('stroke' in d.attributes) {
// There is no proper way to identify the element containing the graph (which we
// don't want a border around), as it is just has tag 'polygon'. Instead we assume
// that the first polygon we see is that element
if (d.tag != 'polygon' || !firstPolygon)
d.attributes.stroke = borderColor;
else
firstPolygon = false;
}
})
.renderDot(resultSet.interpretation.data.dot[this.props.offset]);
};
}

View File

@@ -8,12 +8,14 @@ import {
InterpretedResultsSortState,
ResultSet,
ALERTS_TABLE_NAME,
GRAPH_TABLE_NAME,
SELECT_TABLE_NAME,
getDefaultResultSetName,
ParsedResultSets,
IntoResultsViewMsg,
} from '../pure/interface-types';
import { PathTable } from './alert-table';
import { Graph } from './graph';
import { RawTable } from './raw-results-table';
import {
ResultTableProps,
@@ -107,7 +109,7 @@ export class ResultTables
}
private getInterpretedTableName(): string {
return ALERTS_TABLE_NAME;
return this.props.interpretation?.data.t === 'GraphInterpretationData' ? GRAPH_TABLE_NAME : ALERTS_TABLE_NAME;
}
private getResultSetNames(): string[] {
@@ -354,8 +356,19 @@ class ResultTable extends React.Component<ResultTableProps, Record<string, never
switch (resultSet.t) {
case 'RawResultSet': return <RawTable
{...this.props} resultSet={resultSet} />;
case 'InterpretedResultSet': return <PathTable
{...this.props} resultSet={resultSet} />;
case 'InterpretedResultSet': {
const data = resultSet.interpretation.data;
switch (data.t) {
case 'SarifInterpretationData': {
const sarifResultSet = { ...resultSet, interpretation: { ...resultSet.interpretation, data } };
return <PathTable {...this.props} resultSet={sarifResultSet} />;
}
case 'GraphInterpretationData': {
const grapResultSet = { ...resultSet, interpretation: { ...resultSet.interpretation, data } };
return <Graph {...this.props} resultSet={grapResultSet} />;
}
}
}
}
}
}

View File

@@ -11,6 +11,7 @@ import {
QueryMetadata,
ResultsPaths,
ALERTS_TABLE_NAME,
GRAPH_TABLE_NAME,
ParsedResultSets,
} from '../pure/interface-types';
import { EventHandlers as EventHandlerList } from './event-handler-list';
@@ -105,7 +106,7 @@ class App extends React.Component<Record<string, never>, ResultsViewState> {
void this.loadResults();
break;
case 'showInterpretedPage': {
const tableName = ALERTS_TABLE_NAME;
const tableName = msg.interpretation.data.t === 'GraphInterpretationData' ? GRAPH_TABLE_NAME : ALERTS_TABLE_NAME;
this.updateStateWithNewResultsInfo({
resultsPath: '', // FIXME: Not used for interpreted, refactor so this is not needed
@@ -263,6 +264,8 @@ class App extends React.Component<Record<string, never>, ResultsViewState> {
) {
const parsedResultSets = displayedResults.resultsInfo.parsedResultSets;
const key = (parsedResultSets.selectedTable || '') + parsedResultSets.pageNumber;
const data = displayedResults.resultsInfo.interpretation?.data;
return (
<ResultTables
key={key}
@@ -282,9 +285,7 @@ class App extends React.Component<Record<string, never>, ResultsViewState> {
: undefined
}
sortStates={displayedResults.results.sortStates}
interpretedSortState={
displayedResults.resultsInfo.interpretation?.data.sortState
}
interpretedSortState={data?.t == 'SarifInterpretationData' ? data.sortState : undefined}
isLoadingNewResults={
this.state.isExpectingResultsUpdate ||
this.state.nextResultsInfo !== null

View File

@@ -134,6 +134,14 @@ select {
font-size: inherit;
}
.vscode-codeql__result-tables-graph {
background-color: transparent;
border-color: var(--vscode-dropdown-border);
color: var(--vscode-editor-foreground);
text-align: center;
width: 100%;
}
.vscode-codeql__result-tables-updating-text {
margin-left: 1em;
}

View File

@@ -6,7 +6,7 @@
"module": "commonjs",
"target": "es2017",
"outDir": "out",
"lib": ["ES2020"],
"lib": ["ES2020", "dom"], // No idea what I am doing; needed to avoid `Cannot find name 'RequestInit'` error
"moduleResolution": "node",
"sourceMap": true,
"rootDir": "src",