Compare commits

...

267 Commits

Author SHA1 Message Date
Dave Bartolomeo
d4673d9ca0 Merge pull request #1493 from dbartol/v1.16.12
Some checks failed
Release / Release (push) Has been cancelled
Release / Publish to VS Code Marketplace (push) Has been cancelled
Release / Publish to Open VSX Registry (push) Has been cancelled
v1.16.12
2022-09-01 16:38:14 -04:00
Dave Bartolomeo
87f45a7739 v1.16.12 2022-09-01 16:25:04 -04:00
Koen Vlaswinkel
0c89df9a80 Merge pull request #1482 from github/koesie10/bundle-codicons
Bundle Codicons using Webpack
2022-09-01 16:12:55 +02:00
Koen Vlaswinkel
ba8b32078d Simplify and clarify Webpack font config 2022-09-01 11:21:06 +02:00
Koen Vlaswinkel
fa4dd087e5 Remove Codicons references from webview 2022-09-01 09:39:19 +02:00
Dave Bartolomeo
ac74b967b3 Merge pull request #1490 from dbartol/dbartol/log-version/work 2022-08-31 21:01:36 -04:00
Dave Bartolomeo
c349c6a048 Fix race condition when generating evaluator log summaries
The original code that logged the human-readable log summary generated the log asynchronously, which was a reasonable choice. When I added support for viewing and scanning logs, I didn't notice that the summary was being generated asynchronously, and wrote my code assuming that the summary was already on disk when I opened it to find where each relation's log started. The effect was that, depending on timing, the evaluation sometimes failed with an error popup complaining about not being able to open the log summary file.

The fix is to _generate_ the log summary synchronously, but continue to _log_ it asynchronously.
2022-08-31 18:17:45 -04:00
Dave Bartolomeo
234b05994c Guard --sourcemap option based on CLI version 2022-08-31 18:08:21 -04:00
Koen Vlaswinkel
af8f0231c0 Merge pull request #1485 from github/koesie10/add-github-download-button
Remove canary requirement for GitHub database download
2022-08-31 16:57:12 +02:00
Edoardo Pirovano
84bd029749 Restart CLI server too when restarting query server 2022-08-31 14:39:44 +01:00
shati-patel
7d2e4b6de4 Bump CLI version to 2.10.4 for integration tests 2022-08-31 13:52:40 +01:00
Koen Vlaswinkel
23a0e03cef Completely remove using credentials in non-canary mode
This does not remove the previously added mechanism of not requesting
credentials, but using them when they are available. I expect this to be
used in the future.
2022-08-31 14:22:17 +02:00
Koen Vlaswinkel
21c5ed01ad Fix typo in getOctokit JSDoc
Co-authored-by: Andrew Eisenberg <aeisenberg@github.com>
2022-08-31 11:48:27 +02:00
Koen Vlaswinkel
d2af550bcc Merge remote-tracking branch 'origin/main' into koesie10/bundle-codicons 2022-08-31 09:51:46 +02:00
Koen Vlaswinkel
cf36a52762 Merge pull request #1478 from github/koesie10/abstract-interface-manager
Add abstract interface manager
2022-08-31 09:48:05 +02:00
Koen Vlaswinkel
ac1a97efa0 Refactor databaseFetcher tests to not use proxyquire 2022-08-30 15:32:08 +02:00
Koen Vlaswinkel
8d5067f622 Update CHANGELOG 2022-08-30 15:09:16 +02:00
Koen Vlaswinkel
fe5f1c417d Remove authentication requirement for download GitHub databases
This makes authentication for download GitHub CodeQL databases optional.
If you are already authenticated, your token will be used. If you are
not authenticated, an anonymous request will be made.

If the canary flag is enabled, you will be prompted for credentials when
downloading a database and you are not yet logged in.
2022-08-30 15:05:15 +02:00
Koen Vlaswinkel
95438bb7e3 Remove canary requirement for GitHub database download 2022-08-30 14:33:48 +02:00
Koen Vlaswinkel
6d7d0ca41a Merge pull request #1477 from github/koesie10/unified-webpack-bundle
Unify the Webpack bundle
2022-08-30 11:29:45 +02:00
Koen Vlaswinkel
3749e17769 Bundle Codicons using Webpack
This will include the Codicons inside the webview bundle, reducing the
number of files that need to be loaded and the resource roots that need
to be included.
2022-08-29 14:31:29 +02:00
Koen Vlaswinkel
ee49fb5070 Merge branch 'koesie10/unified-webpack-bundle' into koesie10/abstract-interface-manager 2022-08-29 14:12:20 +02:00
Koen Vlaswinkel
de6c523bad Merge remote-tracking branch 'origin/main' into koesie10/unified-webpack-bundle 2022-08-29 13:57:23 +02:00
Koen Vlaswinkel
6612c279ae Merge pull request #1479 from github/koesie10/improve-controller-repo-prompt
Improve prompot for controller repo
2022-08-29 09:53:26 +02:00
Koen Vlaswinkel
2dfa0e8b52 Simplify interface manager and types 2022-08-29 09:51:49 +02:00
Koen Vlaswinkel
0197306713 Remove unnecessary top-level package-lock.json 2022-08-29 09:47:24 +02:00
Dave Bartolomeo
269165eaa3 Merge pull request #1476 from github/version/bump-to-v1.6.12
Bump version to v1.6.12
2022-08-26 10:38:05 -04:00
Koen Vlaswinkel
14c736d72e Improve prompot for controller repo
This will improve the prompt for the controller repo by making clear
that the GitHub Actions workflow will be run in the specified repo.
2022-08-26 13:58:11 +02:00
Koen Vlaswinkel
b8898b939c Add abstract interface manager
This will add a new abstract class that implements the creation of the
panel and webview to reduce duplication across the different interface
managers.
2022-08-26 12:34:28 +02:00
Koen Vlaswinkel
45da1e0f1f Unify the Webpack bundle
This will move all webviews into a single Webpack bundle. This will make
it easier to add new webviews since we don't need to add a new bundle,
but just need to add a new directory with an `index.tsx` file.

It also moves the CSS processing to Webpack so that we don't need to
specify the CSS files to use separately, but can simply do so in the
TypeScript files.
2022-08-26 11:15:24 +02:00
dbartol
88c990c6ae Bump version to v1.6.12 2022-08-25 20:46:21 +00:00
Dave Bartolomeo
ac7211c117 Merge pull request #1475 from dbartol/dbartol/extension-release/work
Some checks failed
Release / Release (push) Has been cancelled
Release / Publish to VS Code Marketplace (push) Has been cancelled
Release / Publish to Open VSX Registry (push) Has been cancelled
Prepare for release 1.6.11
2022-08-25 16:36:44 -04:00
Dave Bartolomeo
d1d13fbd2e Update changelog for release 2022-08-25 13:11:50 -04:00
Dave Bartolomeo
f99166d26c Update Node version to match vscode 2022-08-25 13:01:35 -04:00
dependabot[bot]
9cd6f9a768 Bump d3 and @types/d3 in /extensions/ql-vscode (#1461)
Bumps [d3](https://github.com/d3/d3) and [@types/d3](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/d3). These dependencies needed to be updated together.

Updates `d3` from 6.7.0 to 7.6.1
- [Release notes](https://github.com/d3/d3/releases)
- [Changelog](https://github.com/d3/d3/blob/main/CHANGES.md)
- [Commits](https://github.com/d3/d3/compare/v6.7.0...v7.6.1)

Updates `@types/d3` from 6.7.5 to 7.4.0
- [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases)
- [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/d3)

---
updated-dependencies:
- dependency-name: d3
  dependency-type: direct:production
  update-type: version-update:semver-major
- dependency-name: "@types/d3"
  dependency-type: direct:development
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-08-24 08:13:04 -07:00
Koen Vlaswinkel
4dd16f4611 Merge pull request #1472 from github/koesie10/fix-data-not-loaded-in-mrva-results
Fix data not being loaded in MRVA results panel
2022-08-24 15:14:26 +02:00
Koen Vlaswinkel
2113d08545 Fix data not being loaded in MRVA results panel
When the MRVA results panel is closed (so the panel gets disposed) and
opened again, it would not load the MRVA data (such as whether a query
has already been downloaded). This fixes it by also resetting the
internal state of whether the panel is loaded when the panel is
disposed.
2022-08-24 13:20:01 +02:00
Koen Vlaswinkel
5b5ef26864 Merge pull request #1471 from github/revert-1466-koesie10/add-github-download-button
Revert "Remove canary requirement for GitHub database download"
2022-08-24 12:12:44 +02:00
Koen Vlaswinkel
c5a6e64df8 Revert "Remove canary requirement for GitHub database download" 2022-08-24 11:51:44 +02:00
Charis Kyriakou
178d626062 Upgrade webview-ui-toolkit version with long link fix (#1469) 2022-08-24 10:17:41 +01:00
Dave Bartolomeo
d1d48b3506 Merge pull request #1468 from dbartol/dbartol/pmod-highlight/work
Add `implements` and `signature` to syntax highlighting
2022-08-23 17:02:03 -04:00
Dave Bartolomeo
9180d1d9fc Fix comment 2022-08-23 16:42:57 -04:00
Dave Bartolomeo
674c5ecbff Add implements and signature to syntax highlighting 2022-08-23 14:56:47 -04:00
Koen Vlaswinkel
edcac6925c Merge pull request #1466 from github/koesie10/add-github-download-button
Remove canary requirement for GitHub database download
2022-08-23 16:24:12 +02:00
Koen Vlaswinkel
c10500c5ea Update CHANGELOG 2022-08-23 14:58:36 +02:00
Koen Vlaswinkel
0832850009 Remove canary requirement for GitHub database download 2022-08-23 14:45:48 +02:00
Alexander Eyers-Taylor
b352830674 Improve startup time (#1465)
* ArchiveFileSystem: Only parse zips once

* CLIServer: Only get version once
2022-08-23 11:05:10 +01:00
Andrew Eisenberg
e913165249 Merge pull request #1463 from github/aeisenberg/bump-timeout 2022-08-17 17:21:31 -07:00
Andrew Eisenberg
ef94bb3d38 Bump telemetry test timeout
This test is failing occasionally on our CI system. Let's see if this
change prevents the failures.
2022-08-17 15:47:30 -07:00
Shati Patel
4d6076c4ea Escape HTML characters when rendering MRVA results as markdown (#1462) 2022-08-17 10:52:36 +01:00
Dave Bartolomeo
43650fde00 Merge pull request #1454 from github/dbartol/join-order
Report suspicious join orders
2022-08-15 14:13:35 -04:00
Angela P Wen
f2c72a67f6 Bump CLI version to 2.10.3 for integration tests (#1460) 2022-08-15 16:41:26 +00:00
Dave Bartolomeo
2b1f3227ce Fix computation of result sizes in IN_LAYER events 2022-08-12 17:00:26 -04:00
Dave Bartolomeo
841f1d3310 Replace console logging to route through problem reporter 2022-08-12 16:43:21 -04:00
Dave Bartolomeo
99756ae63b Fix PR feedback 2022-08-12 16:25:52 -04:00
Dave Bartolomeo
9a2bea39e6 Better handling of missing log data 2022-08-12 16:14:24 -04:00
Dave Bartolomeo
1aab49c719 Specify return type 2022-08-12 16:01:58 -04:00
Dave Bartolomeo
cf925c256f Update extensions/ql-vscode/src/log-insights/log-scanner-service.ts
Co-authored-by: Andrew Eisenberg <aeisenberg@github.com>
2022-08-12 15:50:28 -04:00
Dave Bartolomeo
8383a76e43 Merge branch 'dbartol/join-order' of https://github.com/github/vscode-codeql into dbartol/join-order 2022-08-12 15:41:52 -04:00
Dave Bartolomeo
c6d792f41e Fix PR feedback
Better handling of malformed RA
2022-08-12 15:39:32 -04:00
Dave Bartolomeo
277192e7d3 Update extensions/ql-vscode/src/log-insights/join-order.ts
Co-authored-by: Andrew Eisenberg <aeisenberg@github.com>
2022-08-12 14:59:20 -04:00
Dave Bartolomeo
85988ecf34 Update extensions/ql-vscode/src/log-insights/join-order.ts
Co-authored-by: Andrew Eisenberg <aeisenberg@github.com>
2022-08-12 14:50:10 -04:00
Dave Bartolomeo
49d12674b7 Cache regexprs 2022-08-12 14:47:50 -04:00
Dave Bartolomeo
beeb19dc05 Fix typo 2022-08-12 12:58:46 -04:00
Dave Bartolomeo
de88d27057 Update extensions/ql-vscode/src/log-insights/join-order.ts
Co-authored-by: Andrew Eisenberg <aeisenberg@github.com>
2022-08-12 12:49:29 -04:00
Dave Bartolomeo
eb2d00e999 Update extensions/ql-vscode/src/log-insights/join-order.ts
Co-authored-by: Andrew Eisenberg <aeisenberg@github.com>
2022-08-12 12:48:28 -04:00
Dave Bartolomeo
d58fb54928 Better formatting of metrics 2022-08-11 13:51:11 -04:00
Dave Bartolomeo
fdc209ca08 Test for log scanning 2022-08-10 18:07:59 -04:00
Dave Bartolomeo
28092f2b86 Move more of log scanning into pure code 2022-08-10 17:33:55 -04:00
Dave Bartolomeo
8970ad78ae Remove code added via bad merge 2022-08-10 13:51:08 -04:00
Dave Bartolomeo
e7a0c58940 Fix CodeQL alert 2022-08-10 13:18:00 -04:00
Dave Bartolomeo
02270aaeee Fix lint 2022-08-10 13:13:59 -04:00
Dave Bartolomeo
51fb03b4b1 Fix tests to match code changes 2022-08-10 13:11:34 -04:00
Dave Bartolomeo
838a2b71ac Scan logs on change in current query 2022-08-09 18:02:27 -04:00
Charis Kyriakou
f01c421d42 Merge pull request #1458 from github/version/bump-to-v1.6.11
Bump version to v1.6.11
2022-08-09 16:59:14 +01:00
charisk
561bc6f53c Bump version to v1.6.11 2022-08-09 15:21:26 +00:00
Charis Kyriakou
24b421e82d v1.6.10 (#1456)
Some checks failed
Release / Release (push) Has been cancelled
Release / Publish to VS Code Marketplace (push) Has been cancelled
Release / Publish to Open VSX Registry (push) Has been cancelled
2022-08-09 16:17:57 +01:00
Dave Bartolomeo
3c57597a19 Share code for splitting records from pseudo-JSONL 2022-08-05 17:36:45 -04:00
Dave Bartolomeo
e8d5029912 Merge remote-tracking branch 'origin/main' into dbartol/join-order-temp 2022-08-05 17:34:52 -04:00
Dave Bartolomeo
cb514f5c78 Pre-cleanup to avoid merge conflicts 2022-08-05 14:59:40 -04:00
Dave Bartolomeo
57bb8cee41 Update regexes to match new summary text 2022-08-04 16:17:27 -04:00
Dave Bartolomeo
1219ef4a8c Remove unnecessary command 2022-08-04 16:17:09 -04:00
Dave Bartolomeo
677a0f7940 Fix lint 2022-08-04 14:42:47 -04:00
Charis Kyriakou
b8cca29eb3 Ensure query history state is persisted after new query is added (#1451) 2022-08-04 15:06:47 +01:00
Shati Patel
4cbf104bdf (Minor) Remove outdated comment (#1453) 2022-08-04 13:24:48 +00:00
Angela P Wen
26ccde9e7d Bump CLI version to 2.10.2 for integration tests 2022-08-03 10:30:55 +01:00
Angela P Wen
beb5b78b89 Add 50ms wait for flaky telemetry popup test (#1449) 2022-08-02 08:24:07 -07:00
Dave Bartolomeo
c3a21b93c0 Merge pull request #1430 from github/dbartol/goto-ql
Initial implementation of sourcemap-based jump-to-QL command
2022-08-01 13:52:06 -04:00
Dave Bartolomeo
6b9f73e156 Add comment to test data file 2022-08-01 13:19:15 -04:00
Dave Bartolomeo
6409e09063 Code cleanup 2022-08-01 12:28:35 -04:00
Dave Bartolomeo
8f5611b074 Move sourcemap tests to cli-integration 2022-08-01 12:14:00 -04:00
Dave Bartolomeo
7f3fcce1ac Temporarily increase delay for extension activation in test 2022-07-29 13:11:46 -04:00
Dave Bartolomeo
4bc1d1ed8a Force activation of extension 2022-07-29 12:44:06 -04:00
Dave Bartolomeo
02e5b4e830 Fix installation of dependent extensions 2022-07-29 12:03:43 -04:00
Dave Bartolomeo
538792e8bb Try installing extension dependencies for minimal-workspace tests 2022-07-29 11:35:52 -04:00
Dave Bartolomeo
56ec970121 Merge branch 'dbartol/goto-ql' of https://github.com/github/vscode-codeql into dbartol/goto-ql 2022-07-29 11:01:07 -04:00
Dave Bartolomeo
57a04297bd Only disable specific extensions for minimal-workspace tests 2022-07-29 11:01:02 -04:00
Dave Bartolomeo
59f1e4e90a Update extensions/ql-vscode/src/pure/log-summary-parser.ts 2022-07-28 22:31:18 -04:00
Dave Bartolomeo
7c1fce3319 Merge remote-tracking branch 'origin/main' into dbartol/goto-ql 2022-07-28 22:29:45 -04:00
Dave Bartolomeo
476ea7aef0 Integration test 2022-07-28 22:20:22 -04:00
Elena Tanasoiu
0c654c4320 Merge pull request #1444 from github/elenatanasoiu/fix-bugs
Don't show parentheses when results are not yet fetched in Query History
2022-07-26 10:33:32 +01:00
Elena Tanasoiu
895ac6ae26 Squash extra whitespace for Query History labels
We'd like to remove duplicate whitespace in these labels in order
to make it less likely that we introduce extra space.

We initially also tried trimming whitespaces at the start and end
of these labels but that had no effect.
2022-07-26 09:49:27 +01:00
Elena Tanasoiu
52484f1211 Don't show parentheses when results are not yet fetched
We missed a place where we needed to check if results are present
before attempting to show them.

Let's also add tests for this.
2022-07-26 09:47:55 +01:00
Elena Tanasoiu
cba188b4db Use named arguments for mock function
We'd like to be able to add tests for when the result count exists and
when it's missing.

Let's change the createMockRemoteQueryInfo method so that we can pass
in parameters by name, e.g.

```
createMockRemoteQueryInfo(undefined, 2)
```

becomes

```
createMockRemoteQueryInfo({ repositoryCount: 2 }
```
2022-07-26 09:46:10 +01:00
Elena Tanasoiu
123b1fc085 Clarify title description
To make it clear it's referring to a `user-specified` label, not that the user is not specified.
2022-07-25 17:16:36 +01:00
Angela P Wen
833f8e06ca Add a tree viewer UI for the evaluator logs (#1433)
Co-authored-by: Aditya Sharad <6874315+adityasharad@users.noreply.github.com>
2022-07-22 12:01:39 +02:00
Andrew Eisenberg
747049ed1b Merge pull request #1435 from github/dependabot/npm_and_yarn/extensions/ql-vscode/yargs-parser-20.2.4
Bump yargs-parser from 5.0.0-security.0 to 20.2.4 in /extensions/ql-vscode
2022-07-20 08:47:15 -07:00
Andrew Eisenberg
d62e9181f2 Merge pull request #1436 from github/dependabot/npm_and_yarn/extensions/ql-vscode/semver-regex-and-husky-3.1.4
Bump semver-regex and husky in /extensions/ql-vscode
2022-07-20 08:45:04 -07:00
Dave Bartolomeo
e4d1f4e73e Fix newline handling for cross-platform logs
We were splitting JSONL records based on the current OS newline sequence. In order to handle reading of logs from the opposite OS, I've switched our split to handle both flavors of line ending. This originally showed up as log parser unit tests failing on Windows (the checked-in log used Unix line endings), but could affect real world usage as well.
2022-07-20 11:21:53 -04:00
dependabot[bot]
c1922126d3 Bump terser from 5.14.1 to 5.14.2 in /extensions/ql-vscode
Bumps [terser](https://github.com/terser/terser) from 5.14.1 to 5.14.2.
- [Release notes](https://github.com/terser/terser/releases)
- [Changelog](https://github.com/terser/terser/blob/master/CHANGELOG.md)
- [Commits](https://github.com/terser/terser/commits)

---
updated-dependencies:
- dependency-name: terser
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
2022-07-20 12:03:15 +01:00
elenatanasoiu
d2ebb3d20a Bump version to v1.6.10 2022-07-20 11:57:41 +01:00
Angela P Wen
72858e341a Bump CLI version to 2.10.1 for integration tests (#1442) 2022-07-20 11:55:43 +02:00
Elena Tanasoiu
4499773f6f Merge pull request #1440 from github/v1.6.9
Some checks failed
Release / Release (push) Has been cancelled
Release / Publish to VS Code Marketplace (push) Has been cancelled
Release / Publish to Open VSX Registry (push) Has been cancelled
v1.6.9
2022-07-20 10:16:21 +01:00
Elena Tanasoiu
1d3b0e0ca9 v1.6.9 2022-07-20 10:01:12 +01:00
Elena Tanasoiu
98e503c768 Merge pull request #1438 from github/shati-patel/gist-description
MRVA: Fix Gist description when repository count is undefined
2022-07-20 09:46:22 +01:00
Elena Tanasoiu
62c3974d35 Check for undefined, null or zero repositories
`undefined`, `null` and 0 will evaluate to `false` so if we only want to
display the repository count when these values are not present we can
check for a truthy value:

```
query.repositoryCount ? `(${pluralize(...)})` : '';
```

instead of checking explicitly:

```
query.repositoryCount !== undefined && query.repositoryCount !== null && query.repositoryCount != 0 ? `(${pluralize(...)})` : '';
```
2022-07-20 09:30:54 +01:00
Dave Bartolomeo
40e0027074 Fix newline handling for cross-platform logs
We were splitting JSONL records based on the current OS newline sequence. In order to handle reading of logs from the opposite OS, I've switched our split to handle both flavors of line ending. This originally showed up as log parser unit tests failing on Windows (the checked-in log used Unix line endings), but could affect real world usage as well.
2022-07-19 17:29:33 -04:00
shati-patel
ab1c2e0a0d Explicitly check for undefined 2022-07-19 20:00:10 +01:00
shati-patel
d918c41197 Fix Gist description when repository count is undefined 2022-07-19 18:25:25 +01:00
Dave Bartolomeo
84048ccac1 Merge remote-tracking branch 'origin/main' into dbartol/goto-ql 2022-07-19 09:39:51 -04:00
dependabot[bot]
cbb09da0d0 Bump semver-regex and husky in /extensions/ql-vscode
Bumps [semver-regex](https://github.com/sindresorhus/semver-regex) and [husky](https://github.com/typicode/husky). These dependencies needed to be updated together.

Updates `semver-regex` from 2.0.0 to 3.1.4
- [Release notes](https://github.com/sindresorhus/semver-regex/releases)
- [Commits](https://github.com/sindresorhus/semver-regex/compare/v2.0.0...v3.1.4)

Updates `husky` from 4.2.5 to 4.3.8
- [Release notes](https://github.com/typicode/husky/releases)
- [Commits](https://github.com/typicode/husky/compare/v4.2.5...v4.3.8)

---
updated-dependencies:
- dependency-name: semver-regex
  dependency-type: indirect
- dependency-name: husky
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] <support@github.com>
2022-07-19 12:51:24 +00:00
dependabot[bot]
c8d3428f21 Bump yargs-parser in /extensions/ql-vscode
Bumps [yargs-parser](https://github.com/yargs/yargs-parser) from 5.0.0-security.0 to 20.2.4.
- [Release notes](https://github.com/yargs/yargs-parser/releases)
- [Changelog](https://github.com/yargs/yargs-parser/blob/main/CHANGELOG.md)
- [Commits](https://github.com/yargs/yargs-parser/commits/v20.2.4)

---
updated-dependencies:
- dependency-name: yargs-parser
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
2022-07-19 12:51:22 +00:00
Elena Tanasoiu
2cf5b39cfe Merge pull request #1432 from github/charisk-elena/result-count-on-history-labels
Add result count to remote queries in Query History
2022-07-19 13:50:22 +01:00
Elena Tanasoiu
13921bf8a2 Extract sum method for adding up repo results
When a queryResult is created, it comes with an array for AnalysisSummaries.
There is one summary per repository.

We've had to calculate the total number of results for all summaries in multiple
places, so let's extract a method for this as well.
2022-07-19 13:26:56 +01:00
Elena Tanasoiu
12a97ecba2 Shorten param forwarding for repositoryCount 2022-07-19 13:26:54 +01:00
Elena Tanasoiu
26529232f4 Rename numRepositoriesQueries to repositoryCount
To make it consistent with `resultCount`.
2022-07-19 13:25:48 +01:00
Elena Tanasoiu
1b425fc261 DRY up labels using the new pluralize method 2022-07-19 13:25:40 +01:00
Elena Tanasoiu
9c598c2f06 Extract pluralize method
There are at least 4 different files where this method could DRY things up,
so let's extract it.

I've chosen to move it to src/helpers.ts but happy to be told there's a better
place for shared utility methods like this one.
2022-07-19 12:32:24 +01:00
Elena Tanasoiu
99a784f072 Be able to sort remote queries by number of results
Previously we would set all remote query results to -1 when someone
attempted to sort queries.

We would then only sort local queries as those had access to the number
of results.

Let's include number of results for remote queries in the sorting.

Co-authored-by: Shati Patel <shati-patel@github.com>
2022-07-19 12:32:24 +01:00
Elena Tanasoiu
030488a459 Make local and remote query results match
In the previous commit we're now displaying number of results for remote
queries.

Previously we could only do this for local queries.

Let's make the format match for both types of queries by displaying
number of results in parentheses: `(x results)`.

Co-authored-by: Shati Patel <shati-patel@github.com>
2022-07-19 12:32:24 +01:00
Elena Tanasoiu
377f7965b1 Add result count to remote queries in Query History
When you run a remote query, we'd like to display more information about
it in the Query History panel.

At the moment we've improved this [1] by adding the language and number of repositories.

In this commit we're also adding the number of results for a remote query.

So the final format of the query history item will change from:

`<query_name> - <query_status>`

to

`<query_name> (<language>) on x repositories (y results) - <query_status>`

[1]: https://github.com/github/vscode-codeql/pull/1427

Co-authored-by: Charis Kyriakou <charisk@github.com>
Co-authored-by: Shati Patel <shati-patel@github.com>
2022-07-19 12:32:22 +01:00
Charis Kyriakou
651a6fbda8 Ensure completed flag is set on remote query history items (#1434) 2022-07-19 10:40:02 +01:00
Elena Tanasoiu
55ffdf7963 Merge pull request #1431 from github/shati-elena/rename-gist
Add useful information to MRVA gist titles
2022-07-19 09:11:47 +01:00
Elena Tanasoiu
cc907d2f31 Add test for exportResultsToGist method
While we're here we're also adding a test for the `exportResultsToGist`
method, as there were no tests for the `export-results.ts` file.

We initially attempted to add the test to the pure-tests folder, but the
`export-results.ts` file imports some components from `vscode`, which
meant we needed to set up the test in an environment where VSCode
dependencies are available.

We chose to add the test to `vscode-tests/no-workspace` for convenience,
as there are already other unit tests there.

We've also had to import our own query and analysis result to be able
to work with data closer to reality for exported results.

Since we've introduced functionality to build a gist title, let's check
that the `exportResultsToGist` method will forward the correct title to
the GitHub Actions API.

Co-authored-by: Shati Patel <shati-patel@github.com>
2022-07-18 19:52:51 +01:00
Dave Bartolomeo
49a1576d14 Merge branch 'dbartol/goto-ql' of https://github.com/github/vscode-codeql into dbartol/goto-ql 2022-07-18 14:33:37 -04:00
Dave Bartolomeo
0cc4561ee9 Discard cached sourcemap when summary document is closed
Also some minor lint feedback
2022-07-18 14:33:33 -04:00
Elena Tanasoiu
c4df9dbec8 Extract method for creating Extension context
We'd like to re-use this to test the `exportResultsToGist` method in
`export-results.ts`.

So let's move it to a shared folder in the `vscode-tests/no-workspace` folder.

Since there's no `helper.ts` file in this folder and to avoid any confusion with
the `helpers.test.ts` file, I've opted to put this shared method into `index.ts`.

Happy to be told there's a better pattern for this as it doesn't feel very nice!
2022-07-18 19:22:44 +01:00
Elena Tanasoiu
c384a631dc Handle missing repo count gracefully
Let's handle this case gracefully and skip displaying the number of repositories
when they're not available.

Similarly let's add a check to see if we should pluralize the `repository` noun
or not.

Co-authored-by: Shati Patel <shati-patel@github.com>
2022-07-18 19:22:44 +01:00
Elena Tanasoiu
b079690f0e Add useful information to MRVA gist titles
All exported MRVA gists are given the name `CodeQL variant analysis
results', which makes it hard to work out what it contains at a glance.

We're adding more information in the gist title to make it more useful.

Example of new title:

`Empty Block (Go) x results (y repositories)`

This translates to:

`<query name> (<query language>) <number of results> results (<number of repositories> repositories)`

Co-authored-by: Shati Patel <shati-patel@github.com>
2022-07-18 19:22:41 +01:00
Elena Tanasoiu
4e863e995b Introduce method to add analysis results
We'd like to improve MRVA query gists by giving them more descriptive
titles that contain useful information about the query.

Let's add the number of query results to the title of the gist.

To do this we'll first need to count all the results provided to us in
the `analysisResults` array. There is an item in this array for each of
the repositories we've queried, so we're introducing a method to sum up
results for all the items in the array.

Co-authored-by: Shati Patel <shati-patel@github.com>
2022-07-18 19:20:58 +01:00
Dave Bartolomeo
576737cac8 Update extensions/ql-vscode/src/log-insights/summary-language-support.ts
Co-authored-by: Andrew Eisenberg <aeisenberg@github.com>
2022-07-15 14:50:48 -04:00
Dave Bartolomeo
742aa4ca19 Use error message helper 2022-07-15 11:43:05 -04:00
Shati Patel
f992679e94 MRVA: Include more info in query history label (#1427)
Co-authored-by: Elena Tanasoiu <elenatanasoiu@github.com>
2022-07-15 13:58:45 +01:00
Shati Patel
ffe1704ac0 Replace code paths dropdown with VS Code UI Toolkit (#1429) 2022-07-15 13:04:36 +01:00
Dave Bartolomeo
b5e6700cba Log message on failure to open sourcemap 2022-07-14 18:10:58 -04:00
Dave Bartolomeo
7f5302dc37 fs-extra 2022-07-14 17:39:16 -04:00
Dave Bartolomeo
3ea5524048 Hide "Go to QL Code" behind canary flag 2022-07-14 17:21:52 -04:00
Dave Bartolomeo
1823ae8397 Fix test expectation 2022-07-14 17:03:39 -04:00
Dave Bartolomeo
6dca9ccbeb Fix linter issues 2022-07-14 14:12:10 -04:00
Dave Bartolomeo
f3c2862937 Fix lint error 2022-07-14 14:06:19 -04:00
Dave Bartolomeo
855cb485d5 Initial implementation of sourcemap-based jump-to-QL command 2022-07-14 13:55:46 -04:00
Edoardo Pirovano
bd2dd04ac6 Regularly scrub query history view 2022-07-14 16:59:08 +01:00
Edoardo Pirovano
bbf4a03b03 Fix typo in config parameter name 2022-07-13 16:34:18 +01:00
Shati Patel
f38eb4895d Replace "repository search" filter box with VS Code UI Toolkit (#1424) 2022-07-13 15:13:31 +01:00
Andrew Eisenberg
f559b59ee5 Merge pull request #1420 from github/robertbrignull/api-retry
Add API retries for octokit requests
2022-07-12 08:12:21 -07:00
Angela P Wen
c9d895ea42 Parse summary of evaluator logs into data model (#1405)
Co-authored-by: Aditya Sharad <6874315+adityasharad@users.noreply.github.com>
Co-authored-by: Andrew Eisenberg <aeisenberg@github.com>
2022-07-12 14:04:55 +02:00
Shati Patel
e57bbcb711 Use VSCodeTags instead of Primer Labels in webview (#1421) 2022-07-01 16:21:44 +01:00
Shati Patel
b311991644 MRVA: Fix grammar in pop-up message (#1416) 2022-07-01 12:43:46 +01:00
Robert
825054a271 Use octokit retry module 2022-07-01 11:19:49 +00:00
Robert
f7aa0a5ae5 Install @octokot/plugin-retry 2022-07-01 11:06:22 +00:00
Andrew Eisenberg
f486ccfac6 Merge pull request #1418 from github/aeisenberg/resolve-ml-libs
Resolve ml-queries from directory
2022-06-30 08:56:15 -07:00
Andrew Eisenberg
70f74d3baf Resolve ml-queries from directory
Previously, there was a bug where quick eval queries would crash when
the eval snippet is in a library file.

The problem was that the `codeql resolve queries` command fails when
passed a library file. The fix is to avoid passing the library file at
all. Instead, pass the directory. This is safe because the resolve
queries command only needs to know which query pack the file is
contained in. Passing in the parent directory is the same as passing in
a file in this particular case.
2022-06-30 08:36:55 -07:00
Charis Kyriakou
ebad1844df MRVA: Don't show notification if user aborts firing off a query (#1417) 2022-06-30 14:35:33 +01:00
Charis Kyriakou
a40a2edaf2 Merge pull request #1414 from github/version/bump-to-v1.6.9
Bump version to v1.6.9
2022-06-29 13:17:30 +01:00
charisk
5f3d525ff8 Bump version to v1.6.9 2022-06-29 11:56:36 +00:00
Charis Kyriakou
8f5d88156f Merge pull request #1413 from github/v1.6.8
Some checks failed
Release / Release (push) Has been cancelled
Release / Publish to VS Code Marketplace (push) Has been cancelled
Release / Publish to Open VSX Registry (push) Has been cancelled
v1.6.8
2022-06-29 12:53:51 +01:00
Charis Kyriakou
7c941fe8a8 v1.6.8 2022-06-29 12:42:18 +01:00
Henry Mercer
e9835cb376 Improve changelog note
Co-authored-by: Edoardo Pirovano <6748066+edoardopirovano@users.noreply.github.com>
2022-06-29 12:01:10 +01:00
Henry Mercer
7651a960b1 Add changelog note 2022-06-29 12:01:10 +01:00
Henry Mercer
5b17a84733 Avoid resolve ml-models errors being logged during quick eval
Currently `resolve ml-models` only supports queryspecs, i.e. .ql, .qls,
directory, and query pack specifications. Therefore quick evaluation within
a library isn't
supported.
2022-06-29 12:01:10 +01:00
Andrew Eisenberg
22873a2f3c Invoke codeql pack install after adding a quick query
This ensures the pack lock file is in place after the quick query is
generated.
2022-06-29 10:25:00 +01:00
Andrew Eisenberg
2debadd3bf Update changelog 2022-06-29 10:25:00 +01:00
Charis Kyriakou
6808d7dcaf MRVA: Display alert text even if location is undefined (#1407) 2022-06-29 08:35:56 +01:00
Shati Patel
3480aa5495 Remove older CLI versions from testing matrix (#1410) 2022-06-28 14:07:11 +00:00
Shati Patel
a4d1ad57c7 Bump CLI version for integration tests (#1409) 2022-06-28 13:49:37 +00:00
Robert
628e0e924d Merge pull request #1408 from github/robertbrignull/cutoff_repos
Add cutoff repos and counts to error message
2022-06-28 06:16:18 -07:00
Robert
16077f4124 Add cutoff repos to error message 2022-06-28 12:21:11 +01:00
Charis Kyriakou
e6a68b3223 Add ability to define repo lists in a file outside of settings (#1402) 2022-06-24 16:48:10 +01:00
Charis Kyriakou
539a494914 Only copy repos that have results when copying repo list (#1406) 2022-06-24 14:13:33 +01:00
Charis Kyriakou
9c29c5c9c6 Add ability to create repo list from MRVA results (#1403) 2022-06-24 09:26:12 +01:00
Charis Kyriakou
fd4b6022a9 Refactor: Invert dependency between query history and remote quries managers (#1396) 2022-06-23 13:28:57 +01:00
dependabot[bot]
58bbb59e39 Bump shell-quote from 1.7.2 to 1.7.3 in /extensions/ql-vscode
Bumps [shell-quote](https://github.com/substack/node-shell-quote) from 1.7.2 to 1.7.3.
- [Release notes](https://github.com/substack/node-shell-quote/releases)
- [Changelog](https://github.com/substack/node-shell-quote/blob/master/CHANGELOG.md)
- [Commits](https://github.com/substack/node-shell-quote/compare/v1.7.2...1.7.3)

---
updated-dependencies:
- dependency-name: shell-quote
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
2022-06-23 10:58:47 +01:00
Robert
5cc55530e1 Merge pull request #1399 from github/robertbrignull/skipped_private_repos
Show in log message when repos are filtered out for being private
2022-06-23 02:28:54 -07:00
Robert
3d74dbf48a Update extensions/ql-vscode/src/vscode-tests/no-workspace/remote-queries/run-remote-query.test.ts
Co-authored-by: Andrew Eisenberg <aeisenberg@github.com>
2022-06-23 10:16:37 +01:00
Elena Tanasoiu
b7489d8f66 Merge pull request #1400 from github/elenatanasoiu/check-for-codeql-folder-in-workspace
Fail fast if codeql repo is missing from the workspace
2022-06-23 09:26:40 +01:00
Elena Tanasoiu
e0b2aa9b45 Update docs for running cli-integration tests 2022-06-23 09:13:20 +01:00
Elena Tanasoiu
10b4c15053 Fail fast if codeql CLI is missing from the workspace
In order to run our cli-integration tests, we're required to have a
local copy of the codeql CLI repo. We can then run the tests by running
the `Launch Integration Tests - With CLI` task from inside VS Code.

(See CONTRIBUTING.md for details.)

If we don't have the CLI repo cloned locally or we're not pointing to it
in `launch.json`, we don't get a clear indication of what the problem is.

The tests will still attempt to run.

Let's fail fast instead and add an actionable error message to the output.
2022-06-23 09:13:20 +01:00
Robert
8bc83a336a Show skipped private repos in log message 2022-06-22 17:18:29 +01:00
Elena Tanasoiu
c84b858205 Merge pull request #1397 from github/elenatanasoiu/improve-mrva-controller-feedback
MRVA: Improve experience when controller repo does not exist
2022-06-22 17:03:26 +01:00
Elena Tanasoiu
e5f3a973a0 Improve experience when controller repo does not exist
The controller repo is set via the `codeQL.variantAnalysis.controllerRepo`
setting in VSCode.

While we have validation to check that the repo is not null and the
format of the controller repo is correct: `<owner>/<repo>`, we still
allow you to provide a non-existent repo (e.g. a mispelled one).

When the MRVA request is sent over to the API, it will verify that the
repo exists and return a very generic "Not Found" response.

This will then be logged out in the "Output" tab for VSCode.

We'd like to give users a better indication of what has gone wrong in
this case so we're making the error message more verbose.

Co-authored-by: Charis Kyriakou <charisk@github.com>
Co-authored-by: Shati Patel <shati-patel@github.com>
2022-06-22 16:42:51 +01:00
Andrew Eisenberg
3682f05a42 Merge pull request #1398 from github/aeisenberg/integration-tests-fix
Fix failing integration test
2022-06-21 17:33:54 -07:00
Andrew Eisenberg
eb5ce029ba Fix failing integration test
How did this ever work? It was using an old variant of the
qlpack name.

Also, this commit makes the unhandledRejection handler less
verbose. This gets hit when the tests end and there is a cancellation.
this is not an error.
2022-06-21 17:22:43 -07:00
Charis Kyriakou
0ebff2d6e6 Add ability of running MRVA against a whole org (#1372) 2022-06-21 09:19:07 +01:00
Andrew Eisenberg
d061634fe3 Merge pull request #1379 from github/aeisenberg/fix-bqrs-decode
Fix quoting of string columns in csv
2022-06-20 08:38:36 -07:00
Andrew Eisenberg
6b9410c67e Merge pull request #1388 from github/aeisenberg/fix-flaky-test
Arcane workaround to fix a flaky test
2022-06-20 08:13:22 -07:00
Andrew Eisenberg
8245e54e9c Update extensions/ql-vscode/src/vscode-tests/no-workspace/query-history.test.ts 2022-06-20 08:00:31 -07:00
Shati Patel
8ee744ef0c Bump CLI version used in integration tests (#1394) 2022-06-20 12:02:53 +00:00
Charis Kyriakou
da179b2580 Use VSCodeProgressRing instead of Primer's Spinner (#1392) 2022-06-20 08:39:28 +01:00
Shati Patel
0714f06adc MRVA: Include number of repositories queried in confirmation message (#1393) 2022-06-17 16:15:13 +01:00
Charis Kyriakou
b2906257a1 Remove use of Primer's box component (#1389) 2022-06-17 08:16:51 +01:00
Shati Patel
18097e4676 Allow repo names with periods (#1391) 2022-06-16 17:43:31 +01:00
Charis Kyriakou
efcade84c6 First pass at using VS Code UI toolkit (#1382) 2022-06-16 08:24:42 +00:00
Andrew Eisenberg
7f27375d17 Arcane workaround to fix a flaky test
For an inexplicable reason, the first time the selection
occurs, the value is incorrect. We often miss this error
in our tests if the expectation is reached before the
selection changed event fires.

It seems that the _second_ time the selection changed
event fires, the value is correct.

This change ensures we wait for the second selection change.
And we avoid running expectations until then.e
2022-06-15 15:16:51 -07:00
Andrew Eisenberg
01e1f134be Merge pull request #1361 from github/dependabot/npm_and_yarn/extensions/ql-vscode/glob-promise-4.2.2
Bump glob-promise from 3.4.0 to 4.2.2 in /extensions/ql-vscode
2022-06-15 21:22:15 +02:00
dependabot[bot]
0695b0557f Bump glob-promise from 3.4.0 to 4.2.2 in /extensions/ql-vscode
Bumps [glob-promise](https://github.com/ahmadnassri/node-glob-promise) from 3.4.0 to 4.2.2.
- [Release notes](https://github.com/ahmadnassri/node-glob-promise/releases)
- [Commits](https://github.com/ahmadnassri/node-glob-promise/compare/v3.4.0...v4.2.2)

---
updated-dependencies:
- dependency-name: glob-promise
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2022-06-15 18:24:24 +00:00
Elena Tanasoiu
c63f0c0833 Merge pull request #1387 from github/elenatanasoiu/node-18
Prepare for Node 18 upgrade
2022-06-15 19:23:01 +01:00
Elena Tanasoiu
3264ffaaa4 Upgrade webpack
We're upgrading the minimum version of webpack from 5.28.0 to 5.62.2
since this version doesn't rely on OpenSSL for its hashing algorithm so
it wouldn't need legacy OpenSSL support when we decide to upgrade to
Node 18.

This allows us to build our extension on Node 18:
https://github.com/github/vscode-codeql/runs/6904100934?check_suite_focus=true

Happily, this also works fine with our current version of Node (16.13.0).
2022-06-15 18:00:46 +01:00
Elena Tanasoiu
40959c8876 Use source-map 0.7.4
A new release of source-map was pushed 10 days ago:
https://github.com/mozilla/source-map/releases/tag/v0.7.4

It contains a fix for building on Node 18 (which was added in Oct
2020): https://github.com/mozilla/source-map/issues/423.

Let's make use of it!
2022-06-15 18:00:38 +01:00
Elena Tanasoiu
ecea7f4638 Merge pull request #1386 from github/elenatanasoiu/update-release-docs
Follow guidance for git tagging in contribution docs
2022-06-15 17:27:27 +01:00
Elena Tanasoiu
0b15a166fa Follow guidance for git tagging in contribution docs
Adding two things:
- A bit more detail on how to add a tag and how to delete a badly named one
- Switch to the official way of sharing tags according to the git docs[^1]

[^1]: https://git-scm.com/book/en/v2/Git-Basics-Tagging
2022-06-15 16:57:38 +01:00
elenatanasoiu
c368424a15 Bump version to v1.6.8 2022-06-15 15:42:36 +01:00
Elena Tanasoiu
5df1f80307 Merge pull request #1384 from github/v1.6.7
Some checks failed
Release / Release (push) Has been cancelled
Release / Publish to VS Code Marketplace (push) Has been cancelled
Release / Publish to Open VSX Registry (push) Has been cancelled
v1.6.7
2022-06-15 15:03:00 +01:00
Elena Tanasoiu
4b59045149 v1.6.7 2022-06-15 13:41:11 +00:00
Andrew Eisenberg
a3a05131c7 Handle quote escaping in csv export 2022-06-13 11:17:37 -07:00
Elena Tanasoiu
a9922b86fe Merge pull request #1374 from github/elenatanasoiu/set-node-version
Use the same Node version as VSCode
2022-06-13 16:54:10 +02:00
Elena Tanasoiu
431350ac0e Merge pull request #1375 from github/elenatanasoiu/adjust-font-on-results
Use base styling for MRVA results font
2022-06-13 16:53:32 +02:00
Elena Tanasoiu
5f8802fe7f Use base styling for MRVA results font
It's been pointed out that MRVA results are hard to read as the font is
small and narrowly spaced. It also doesn't match the font type normally
used in source files.

We can instead switch to using the font-family used by VS Code itself and
increase the font size from `x-small` to `small` for code snippets.
2022-06-13 14:28:15 +01:00
Elena Tanasoiu
5f21594d23 Provide a maximum node version in package.json
As recommended here https://github.com/github/vscode-codeql/pull/1369#issuecomment-1142418037, since the current build for this extension does not work with Node v18 https://github.com/github/vscode-codeql/issues/1373, it would be good to set a maximum node version until this gets addressed.

So we're updating `engines` here to allow for a maximum version, which in this case is v17.0.0.
2022-06-13 11:21:29 +01:00
Elena Tanasoiu
8964ec1a4d Use the same Node version as VSCode
As recommended here https://github.com/github/vscode-codeql/pull/1369#issuecomment-1142418037, we want to stay in sync with the current node version shipped with
VSCode (v16.13.0):

32d40cf44e/remote/.yarnrc (L2)

For this we can add a `.nvmrc` file to alert nvm to switch to the preferred version automatically.

It will also help prevent builds from failing when setting up the project for the first time, as building the extension currently fails in Node v18: https://github.com/github/vscode-codeql/issues/1373

We're also updating the docs to mention using `nvm` to manage node versions and point to the right place to check for current supported versions.
2022-06-13 11:21:25 +01:00
Andrew Eisenberg
aa270e57ec Refactor exportCsvResults and create test
1. `exportCsvResults` now no longer requires an `onFinish` callback.
2. The test adds a generic framework for creating a mock cli server.
   This should be used in future tests.
2022-06-06 10:21:12 +02:00
Andrew Eisenberg
fe7eb07f39 Don't choose a non-existent result set for csv viewing
If the `#select` resultset doesn't exist, arbitrarily choose the first
result set when viewing csv results. This will almost certainly be the
correct result set.

In the future, we could offer a popup if there are multiple result sets
available, but let's wait on that until someone actually asks for it.
2022-06-06 09:23:40 +02:00
Andrew Eisenberg
c10da7f960 Update Changelog 2022-06-03 16:26:02 -07:00
Andrew Eisenberg
0c8390c094 Fix quoting of string columns in csv 2022-06-03 16:24:10 -07:00
shati-patel
d41c63bf7d Change parameter type + extract local variable 2022-06-01 21:19:35 +01:00
shati-patel
a3bbdafabb Add tests for 'tryGetRemoteLocation' 2022-06-01 21:19:35 +01:00
shati-patel
a78eef464b Handle undefined URLs 2022-06-01 21:19:35 +01:00
shati-patel
e8348ac12a Check format of URI 2022-06-01 21:19:35 +01:00
shati-patel
5efc3835db Add sourceLocationPrefix to one of the test files 2022-06-01 21:19:35 +01:00
shati-patel
c4ed6e88de Pass sourceLocationPrefix down through all the functions 2022-06-01 21:19:35 +01:00
shati-patel
51e6559145 Update tryGetRemoteLocation to use sourceLocationPrefix (if available) 2022-06-01 21:19:35 +01:00
Charis Kyriakou
db8b419885 Combine time constants (#1371) 2022-06-01 16:52:18 +01:00
Elena Tanasoiu
475d7cc535 Merge pull request #1369 from github/elenatanasoiu/fix-casing-query-history
Be consistent about casing in Query History menu
2022-06-01 12:24:29 +01:00
Elena Tanasoiu
1858de5ed0 Update Changelog 2022-06-01 12:07:13 +01:00
Angela P Wen
642f4788fb Update tests to CLI v2.9.3 (#1370) 2022-05-31 17:44:43 +00:00
Elena Tanasoiu
7e70f8b758 Be consistent about casing in Query History
Reported here: https://github.com/github/code-scanning/issues/6008

We originally started out by capitalizing each word [1], but made some
small changes [2] which resulted in our Query History options
being inconsistent.

Let's fix that.

[1]: a5da556496/extensions/ql-vscode/package.json
[1]: b470e41431
2022-05-31 18:34:37 +01:00
Charis Kyriakou
e417bea948 Move time constants to time module (#1368) 2022-05-31 13:21:45 +01:00
Andrew Eisenberg
6b4be93169 Merge pull request #1363 from github/aeisenberg/resolve-ml-model
Add new support for resolve ml-models
2022-05-30 18:46:53 -07:00
Andrew Eisenberg
061eaad743 Update extensions/ql-vscode/src/cli.ts
Change version where precise ml-model resolution was introduced.
2022-05-30 18:32:11 -07:00
Andrew Eisenberg
8ff21d6c89 Merge pull request #1365 from github/aeisenberg/time
Extract time functions
2022-05-30 07:54:18 -07:00
Andrew Eisenberg
0d9f4e8c0f Merge pull request #1366 from github/aeisenberg/handle-missing-nwo
Handle missing nwos returned from graphql query
2022-05-30 07:53:55 -07:00
Andrew Eisenberg
02288718dc Handle missing nwos returned from graphql query 2022-05-27 13:12:49 -07:00
Andrew Eisenberg
615cf86fc0 Refactor time functions
Rename, add comments, and extract some local variables.
2022-05-27 08:51:14 -07:00
Andrew Eisenberg
d63a209674 Make conditional statement more explicit 2022-05-27 07:08:58 -07:00
Andrew Eisenberg
9d26304f7a Extract time functions
Create the `time.ts` module as a place to put fime functions.
Move two time functions there and create tests for them.

The `humanizeUnit` function now uses ECMAscript apis. This ensures
that pluralization happens appropriately.

Also, fix a small bug in the results view to enure `repository`
is correctly pluralized.
2022-05-26 15:47:03 -07:00
Andrew Eisenberg
f73bda438a Merge pull request #1362 from github/aeisenberg/last-update-sort
Add sort MRVA results by last updated
2022-05-26 09:15:37 -07:00
Andrew Eisenberg
19b65a654e Fix method name 2022-05-26 08:44:04 -07:00
Andrew Eisenberg
770127e67a Use the repo push icon 2022-05-26 06:55:12 -07:00
Andrew Eisenberg
f373e6467a Store LastUpdated as a duration, not a timestamp
The `lastUpdated` value is now the duration between timestamp of the
last time the repo was updated and time the file was downloaded.
This fixes the duration and it won't change over time.
2022-05-25 20:30:28 -07:00
Andrew Eisenberg
e43b4e66a1 Add sort MRVA results by last updated
1. Refactor references of `Stargazers` to `RepositoryMetadata` since
   the query is now more generic.
2. Update the graphql query to request last updated as well as stars
3. Update web view to display last updated
4. Update sort mechanism for last updated

A few notes:

1. I used `Intl.RelativeTimeFormat` to humanize the times. It wasn't as
   simple as I had hoped since I need to also make a guess as to which
   unit to use.
2. The icon used by last updated is not quite what is in the wireframes.
   But, I wanted to stick with primer icons and I used the closest I can
   get.
3. The last updated time is retrieved when the query is first loaded
   into vscode and then never changes. However, this time is always
   compared with `Date.now()`. So, opening the query up a week from now,
   all of the last updated times would be one week older (even if the
   repository has been updated since then).

   I don't want to re-retrieve the last updated time each time we open
   the query, so this timestamp will get out of date eventually.

   Is this confusing as it is?
2022-05-24 19:57:40 -07:00
Andrew Eisenberg
90ec003386 Add new support for resolve ml-models
The new support will be available in the next
release of the CLI, most likely 2.9.3,

This change requires the query to be run to be
passed in to the call to resolve ml-models.
2022-05-24 17:24:46 -07:00
Angela P Wen
2f9aca785e Log most expensive predicates and timings to query log (#1349) 2022-05-20 13:21:33 -07:00
Andrew Eisenberg
405a6c9901 Merge pull request #1353 from github/aeisenberg/sort-remote-results
Add sorting to variant analysis results
2022-05-20 09:23:10 -07:00
Andrew Eisenberg
3611b1fe61 Add comments and simplify some JSX
Use `ActionMenu.Anchor` instead of `ActionMenu.Button`.

The theming styles are not correct. Will work on that next.
2022-05-20 08:01:54 -07:00
Andrew Eisenberg
7b33441519 Merge pull request #1360 from github/dependabot/npm_and_yarn/extensions/ql-vscode/mocha-10.0.0
Bump mocha from 9.1.3 to 10.0.0 in /extensions/ql-vscode
2022-05-19 08:29:52 -07:00
dependabot[bot]
2a8f61dfbe Bump mocha from 9.1.3 to 10.0.0 in /extensions/ql-vscode
Bumps [mocha](https://github.com/mochajs/mocha) from 9.1.3 to 10.0.0.
- [Release notes](https://github.com/mochajs/mocha/releases)
- [Changelog](https://github.com/mochajs/mocha/blob/master/CHANGELOG.md)
- [Commits](https://github.com/mochajs/mocha/compare/v9.1.3...v10.0.0)

---
updated-dependencies:
- dependency-name: mocha
  dependency-type: direct:development
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2022-05-19 13:05:48 +00:00
Andrew Eisenberg
dcfd6d43c0 Merge pull request #1307 from github/dependabot/npm_and_yarn/extensions/ql-vscode/js-yaml-4.1.0
Bump js-yaml from 3.14.0 to 4.1.0 in /extensions/ql-vscode
2022-05-18 15:03:36 -07:00
Andrew Eisenberg
4e4d8b2f04 Fix js-yaml issues
With js-yaml 4.0, safeLoad is no longer available. Use load instead.
2022-05-18 14:45:28 -07:00
Andrew Eisenberg
50197ba7b7 Merge pull request #1308 from github/dependabot/npm_and_yarn/extensions/ql-vscode/style-loader-3.3.1
Bump style-loader from 0.23.1 to 3.3.1 in /extensions/ql-vscode
2022-05-18 14:40:50 -07:00
Andrew Eisenberg
6c376d8721 Add integration test for ensuring the graphql query succeeds 2022-05-18 14:20:24 -07:00
Andrew Eisenberg
82ada54103 Add sorting to variant analysis results
Sort by stars, number of results, and name.

This also includes a graphql query that retrieves all the stars
for relevant repositories.
2022-05-18 13:56:17 -07:00
shati-patel
0fdfeb3cd3 Bump version to v1.6.7 2022-05-17 16:37:42 +01:00
dependabot[bot]
c32b53613d Bump js-yaml from 3.14.0 to 4.1.0 in /extensions/ql-vscode
Bumps [js-yaml](https://github.com/nodeca/js-yaml) from 3.14.0 to 4.1.0.
- [Release notes](https://github.com/nodeca/js-yaml/releases)
- [Changelog](https://github.com/nodeca/js-yaml/blob/master/CHANGELOG.md)
- [Commits](https://github.com/nodeca/js-yaml/compare/3.14.0...4.1.0)

---
updated-dependencies:
- dependency-name: js-yaml
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2022-05-09 21:51:58 +00:00
Dave Bartolomeo
cff235c420 Auto-format 2022-05-03 18:14:03 -04:00
Dave Bartolomeo
1089a052ec Initial implementation of join order metric scanning 2022-05-03 13:20:30 -04:00
Dave Bartolomeo
1d195cb347 Merge remote-tracking branch 'origin/main' into dbartol/join-order 2022-04-27 17:50:50 -04:00
Dave Bartolomeo
8d8ed28aea Add necessary dependencies 2022-04-27 17:50:46 -04:00
dependabot[bot]
46922de3c0 Bump style-loader from 0.23.1 to 3.3.1 in /extensions/ql-vscode
Bumps [style-loader](https://github.com/webpack-contrib/style-loader) from 0.23.1 to 3.3.1.
- [Release notes](https://github.com/webpack-contrib/style-loader/releases)
- [Changelog](https://github.com/webpack-contrib/style-loader/blob/master/CHANGELOG.md)
- [Commits](https://github.com/webpack-contrib/style-loader/compare/v0.23.1...v3.3.1)

---
updated-dependencies:
- dependency-name: style-loader
  dependency-type: direct:development
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2022-04-21 13:02:49 +00:00
159 changed files with 25513 additions and 4103 deletions

View File

@@ -22,7 +22,7 @@ jobs:
- uses: actions/setup-node@v1
with:
node-version: '16.13.0'
node-version: '16.14.0'
- name: Install dependencies
working-directory: extensions/ql-vscode
@@ -82,7 +82,7 @@ jobs:
- uses: actions/setup-node@v1
with:
node-version: '16.13.0'
node-version: '16.14.0'
- name: Install dependencies
working-directory: extensions/ql-vscode
@@ -118,6 +118,8 @@ jobs:
- name: Run integration tests (Linux)
if: matrix.os == 'ubuntu-latest'
working-directory: extensions/ql-vscode
env:
VSCODE_CODEQL_GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
run: |
sudo apt-get install xvfb
/usr/bin/xvfb-run npm run integration
@@ -125,6 +127,8 @@ jobs:
- name: Run integration tests (Windows)
if: matrix.os == 'windows-latest'
working-directory: extensions/ql-vscode
env:
VSCODE_CODEQL_GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
run: |
npm run integration
@@ -135,7 +139,7 @@ jobs:
strategy:
matrix:
os: [ubuntu-latest, windows-latest]
version: ['v2.3.3', 'v2.4.6', 'v2.5.9', 'v2.6.3', 'v2.7.6', 'v2.8.5', 'v2.9.2', 'nightly']
version: ['v2.6.3', 'v2.7.6', 'v2.8.5', 'v2.9.4', 'v2.10.4', 'nightly']
env:
CLI_VERSION: ${{ matrix.version }}
NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }}
@@ -147,7 +151,7 @@ jobs:
- uses: actions/setup-node@v1
with:
node-version: '16.13.0'
node-version: '16.14.0'
- name: Install dependencies
working-directory: extensions/ql-vscode
@@ -168,9 +172,6 @@ jobs:
if [[ "${{ matrix.version }}" == "nightly" ]]
then
REF="codeql-cli/latest"
elif [[ "${{ matrix.version }}" == "v2.2.6" || "${{ matrix.version }}" == "v2.3.3" ]]
then
REF="codeql-cli/v2.4.5"
else
REF="codeql-cli/${{ matrix.version }}"
fi

View File

@@ -22,7 +22,7 @@ jobs:
- uses: actions/setup-node@v1
with:
node-version: '16.13.0'
node-version: '16.14.0'
- name: Install dependencies
run: |

View File

@@ -29,7 +29,9 @@ Here are a few things you can do that will increase the likelihood of your pull
## Setting up a local build
Make sure you have installed recent versions of vscode (>= v1.52), node (>=12.16), and npm (>= 7.5.2). Earlier versions will probably work, but we no longer test against them.
Make sure you have installed recent versions of vscode, node, and npm. Check the `engines` block in [`package.json`](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/package.json) file for compatible versions. Earlier versions may work, but we no longer test against them.
To automatically switch to the correct version of node, we recommend using [nvm](https://github.com/nvm-sh/nvm), which will pick-up the node version from `.nvmrc`.
### Installing all packages
@@ -93,15 +95,21 @@ Running from a terminal, you _must_ set the `TEST_CODEQL_PATH` variable to point
### Running the integration tests
The _Launch Integration Tests - With CLI_ tests require a CLI instance in order to run. There are several environment variables you can use to configure this.
You will need to run CLI tests using a task from inside of VS Code called _Launch Integration Tests - With CLI_.
From inside of VSCode, open the `launch.json` file and in the _Launch Integration Tests - With CLI_ uncomment and change the environment variables appropriate for your purpose.
The CLI integration tests require the CodeQL standard libraries in order to run so you will need to clone a local copy of the `github/codeql` repository.
From inside of VSCode, open the `launch.json` file and in the _Launch Integration Tests - With CLI_ task, uncomment the `"${workspaceRoot}/../codeql"` line. If necessary, replace value with a path to your checkout, and then run the task.
## Releasing (write access required)
1. Double-check the `CHANGELOG.md` contains all desired change comments and has the version to be released with date at the top.
* Go through all recent PRs and make sure they are properly accounted for.
* Make sure all changelog entries have links back to their PR(s) if appropriate.
1. Double-check that the node version we're using matches the one used for VS Code. If it doesn't, you will then need to update the node version in the following files:
* `.nvmrc` - this will enable `nvm` to automatically switch to the correct node version when you're in the project folder
* `.github/workflows/main.yml` - all the "node-version: <version>" settings
* `.github/workflows/release.yml` - the "node-version: <version>" setting
1. Double-check that the extension `package.json` and `package-lock.json` have the version you intend to release. If you are doing a patch release (as opposed to minor or major version) this should already be correct.
1. Create a PR for this release:
* This PR will contain any missing bits from steps 1 and 2. Most of the time, this will just be updating `CHANGELOG.md` with today's date.
@@ -109,15 +117,34 @@ From inside of VSCode, open the `launch.json` file and in the _Launch Integratio
* Create a new commit with a message the same as the branch name.
* Create a PR for this branch.
* Wait for the PR to be merged into `main`
1. Trigger a release build on Actions by adding a new tag on branch `main` named after the release, as above. Note that when you push to upstream, you will need to fully qualify the ref. A command like this will work:
1. Switch to `main` and add a new tag on the `main` branch with your new version (named after the release), e.g.
```bash
git checkout main
git tag v1.3.6
```
If you've accidentally created a badly named tag, you can delete it via
```bash
git tag -d badly-named-tag
```
1. Push the new tag up:
a. If you're using a fork of the repo:
```bash
git push upstream refs/tags/v1.3.6
```
b. If you're working straight in this repo:
```bash
git push origin refs/tags/v1.3.6
```
This will trigger [a release build](https://github.com/github/vscode-codeql/releases) on Actions.
* **IMPORTANT** Make sure you are on the `main` branch and your local checkout is fully updated when you add the tag.
* If you accidentally add the tag to the wrong ref, you can just force push it to the right one later.
1. Monitor the status of the release build in the `Release` workflow in the Actions tab.
1. Download the VSIX from the draft GitHub release at the top of [the releases page](https://github.com/github/vscode-codeql/releases) that is created when the release build finishes.
1. Unzip the `.vsix` and inspect its `package.json` to make sure the version is what you expect,

View File

@@ -0,0 +1 @@
v16.14.0

View File

@@ -1,5 +1,34 @@
# CodeQL for Visual Studio Code: Changelog
## 1.6.12 - 1 September 2022
- Add ability for users to download databases directly from GitHub. [#1485](https://github.com/github/vscode-codeql/pull/1485)
- Fix a race condition that could cause a failure to open the evaluator log when running a query. [#1490](https://github.com/github/vscode-codeql/pull/1490)
- Fix an error when running a query with an older version of the CodeQL CLI. [#1490](https://github.com/github/vscode-codeql/pull/1490)
## 1.6.11 - 25 August 2022
No user facing changes.
## 1.6.10 - 9 August 2022
No user facing changes.
## 1.6.9 - 20 July 2022
No user facing changes.
## 1.6.8 - 29 June 2022
- Fix a bug where quick queries cannot be compiled if the core libraries are not in the workspace. [#1411](https://github.com/github/vscode-codeql/pull/1411)
- Fix a bug where quick evaluation of library files would display an error message when using CodeQL CLI v2.10.0. [#1412](https://github.com/github/vscode-codeql/pull/1412)
## 1.6.7 - 15 June 2022
- Prints end-of-query evaluator log summaries to the Query Log. [#1349](https://github.com/github/vscode-codeql/pull/1349)
- Be consistent about casing in Query History menu. [#1369](https://github.com/github/vscode-codeql/pull/1369)
- Fix quoting string columns in exported CSV results. [#1379](https://github.com/github/vscode-codeql/pull/1379)
## 1.6.6 - 17 May 2022
No user facing changes.

View File

@@ -1,5 +1,5 @@
import * as gulp from 'gulp';
import { compileTypeScript, watchTypeScript, copyViewCss, cleanOutput, watchCss } from './typescript';
import { compileTypeScript, watchTypeScript, cleanOutput } from './typescript';
import { compileTextMateGrammar } from './textmate';
import { copyTestData } from './tests';
import { compileView, watchView } from './webpack';
@@ -10,7 +10,7 @@ export const buildWithoutPackage =
gulp.series(
cleanOutput,
gulp.parallel(
compileTypeScript, compileTextMateGrammar, compileView, copyTestData, copyViewCss
compileTypeScript, compileTextMateGrammar, compileView, copyTestData
)
);
@@ -23,6 +23,5 @@ export {
copyTestData,
injectAppInsightsKey,
compileView,
watchCss
};
export default gulp.series(buildWithoutPackage, injectAppInsightsKey, packageExtension);

View File

@@ -219,14 +219,14 @@ function transformFile(yaml: any) {
}
export function transpileTextMateGrammar() {
return through.obj((file: Vinyl, _encoding: string, callback: Function): void => {
return through.obj((file: Vinyl, _encoding: string, callback: (err: string | null, file: Vinyl | PluginError) => void): void => {
if (file.isNull()) {
callback(null, file);
}
else if (file.isBuffer()) {
const buf: Buffer = file.contents;
const yamlText: string = buf.toString('utf8');
const jsonData: any = jsYaml.safeLoad(yamlText);
const jsonData: any = jsYaml.load(yamlText);
transformFile(jsonData);
file.contents = Buffer.from(JSON.stringify(jsonData, null, 2), 'utf8');

View File

@@ -39,13 +39,3 @@ export function compileTypeScript() {
export function watchTypeScript() {
gulp.watch('src/**/*.ts', compileTypeScript);
}
export function watchCss() {
gulp.watch('src/**/*.css', copyViewCss);
}
/** Copy CSS files for the results view into the output directory. */
export function copyViewCss() {
return gulp.src('src/**/view/*.css')
.pipe(gulp.dest('out'));
}

View File

@@ -1,12 +1,11 @@
import * as path from 'path';
import * as webpack from 'webpack';
import * as MiniCssExtractPlugin from 'mini-css-extract-plugin';
export const config: webpack.Configuration = {
mode: 'development',
entry: {
resultsView: './src/view/results.tsx',
compareView: './src/compare/view/Compare.tsx',
remoteQueriesView: './src/remote-queries/view/RemoteQueries.tsx',
webview: './src/view/webview.tsx'
},
output: {
path: path.resolve(__dirname, '..', 'out'),
@@ -31,9 +30,7 @@ export const config: webpack.Configuration = {
{
test: /\.less$/,
use: [
{
loader: 'style-loader'
},
MiniCssExtractPlugin.loader,
{
loader: 'css-loader',
options: {
@@ -53,17 +50,31 @@ export const config: webpack.Configuration = {
{
test: /\.css$/,
use: [
{
loader: 'style-loader'
},
MiniCssExtractPlugin.loader,
{
loader: 'css-loader'
}
]
},
{
test: /\.(woff(2)?|ttf|eot)$/,
use: [
{
loader: 'file-loader',
options: {
name: '[name].[ext]',
outputPath: 'fonts/',
// We need this to make Webpack use the correct path for the fonts.
// Without this, the CSS file will use `url([object Module])`
esModule: false
}
},
],
}
]
},
performance: {
hints: false
}
},
plugins: [new MiniCssExtractPlugin()],
};

File diff suppressed because it is too large Load Diff

View File

@@ -4,7 +4,7 @@
"description": "CodeQL for Visual Studio Code",
"author": "GitHub",
"private": true,
"version": "1.6.6",
"version": "1.6.12",
"publisher": "GitHub",
"license": "MIT",
"icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -14,7 +14,7 @@
},
"engines": {
"vscode": "^1.59.0",
"node": ">=16.11.25",
"node": "^16.13.0",
"npm": ">=7.20.6"
},
"categories": [
@@ -35,9 +35,11 @@
},
"activationEvents": [
"onLanguage:ql",
"onLanguage:ql-summary",
"onView:codeQLDatabases",
"onView:codeQLQueryHistory",
"onView:codeQLAstViewer",
"onView:codeQLEvalLogViewer",
"onView:test-explorer",
"onCommand:codeQL.checkForUpdatesToCLI",
"onCommand:codeQL.authenticateToGitHub",
@@ -110,6 +112,12 @@
"extensions": [
".qhelp"
]
},
{
"id": "ql-summary",
"filenames": [
"evaluator-log.summary"
]
}
],
"grammars": [
@@ -224,7 +232,7 @@
},
"codeQL.queryHistory.format": {
"type": "string",
"default": "%q on %d - %s, %r [%t]",
"default": "%q on %d - %s %r [%t]",
"markdownDescription": "Default string for how to label query history items.\n* %t is the time of the query\n* %q is the human-readable query name\n* %f is the query file name\n* %d is the database name\n* %r is the number of results\n* %s is a status string"
},
"codeQL.queryHistory.ttl": {
@@ -280,7 +288,7 @@
"default": "",
"pattern": "^$|^(?:[a-zA-Z0-9]+-)*[a-zA-Z0-9]+/[a-zA-Z0-9-_]+$",
"patternErrorMessage": "Please enter a valid GitHub repository",
"markdownDescription": "[For internal use only] The name of the GitHub repository where you can view the progress and results of the \"Run Variant Analysis\" command. The repository should be of the form `<owner>/<repo>`)."
"markdownDescription": "[For internal use only] The name of the GitHub repository in which the GitHub Actions workflow is run when using the \"Run Variant Analysis\" command. The repository should be of the form `<owner>/<repo>`)."
}
}
},
@@ -471,7 +479,7 @@
},
{
"command": "codeQLQueryHistory.openQuery",
"title": "Open the query that produced these results",
"title": "Open the Query that Produced these Results",
"icon": {
"light": "media/light/edit.svg",
"dark": "media/dark/edit.svg"
@@ -523,15 +531,19 @@
},
{
"command": "codeQLQueryHistory.openQueryDirectory",
"title": "Open query directory"
"title": "Open Query Directory"
},
{
"command": "codeQLQueryHistory.showEvalLog",
"title": "Show Evaluator Log (Raw)"
"title": "Show Evaluator Log (Raw JSON)"
},
{
"command": "codeQLQueryHistory.showEvalLogSummary",
"title": "Show Evaluator Log (Summary)"
"title": "Show Evaluator Log (Summary Text)"
},
{
"command": "codeQLQueryHistory.showEvalLogViewer",
"title": "Show Evaluator Log (UI)"
},
{
"command": "codeQLQueryHistory.cancel",
@@ -573,6 +585,10 @@
"command": "codeQLQueryHistory.openOnGithub",
"title": "Open Variant Analysis on GitHub"
},
{
"command": "codeQLQueryHistory.copyRepoList",
"title": "Copy Repository List"
},
{
"command": "codeQLQueryResults.nextPathStep",
"title": "CodeQL: Show Next Step on Path"
@@ -604,6 +620,19 @@
"light": "media/light/clear-all.svg",
"dark": "media/dark/clear-all.svg"
}
},
{
"command": "codeQLEvalLogViewer.clear",
"title": "Clear Viewer",
"icon": {
"light": "media/light/clear-all.svg",
"dark": "media/dark/clear-all.svg"
}
},
{
"command": "codeQL.gotoQL",
"title": "CodeQL: Go to QL Code",
"enablement": "codeql.hasQLSource"
}
],
"menus": {
@@ -635,7 +664,7 @@
},
{
"command": "codeQLDatabases.chooseDatabaseGithub",
"when": "config.codeQL.canary && view == codeQLDatabases",
"when": "view == codeQLDatabases",
"group": "navigation"
},
{
@@ -677,6 +706,11 @@
"command": "codeQLAstViewer.clear",
"when": "view == codeQLAstViewer",
"group": "navigation"
},
{
"command": "codeQLEvalLogViewer.clear",
"when": "view == codeQLEvalLogViewer",
"group": "navigation"
}
],
"view/item/context": [
@@ -750,6 +784,11 @@
"group": "9_qlCommands",
"when": "codeql.supportsEvalLog && viewItem == rawResultsItem || codeql.supportsEvalLog && viewItem == interpretedResultsItem || codeql.supportsEvalLog && viewItem == cancelledResultsItem"
},
{
"command": "codeQLQueryHistory.showEvalLogViewer",
"group": "9_qlCommands",
"when": "config.codeQL.canary && codeql.supportsEvalLog && viewItem == rawResultsItem || config.codeQL.canary && codeql.supportsEvalLog && viewItem == interpretedResultsItem || config.codeQL.canary && codeql.supportsEvalLog && viewItem == cancelledResultsItem"
},
{
"command": "codeQLQueryHistory.showQueryText",
"group": "9_qlCommands",
@@ -790,6 +829,11 @@
"group": "9_qlCommands",
"when": "viewItem == remoteResultsItem || viewItem == inProgressRemoteResultsItem || viewItem == cancelledResultsItem"
},
{
"command": "codeQLQueryHistory.copyRepoList",
"group": "9_qlCommands",
"when": "viewItem == remoteResultsItem"
},
{
"command": "codeQLTests.showOutputDifferences",
"group": "qltest@1",
@@ -882,10 +926,6 @@
"command": "codeQL.viewCfg",
"when": "resourceScheme == codeql-zip-archive && config.codeQL.canary"
},
{
"command": "codeQL.chooseDatabaseGithub",
"when": "config.codeQL.canary"
},
{
"command": "codeQLDatabases.setCurrentDatabase",
"when": "false"
@@ -966,6 +1006,10 @@
"command": "codeQLQueryHistory.showEvalLogSummary",
"when": "false"
},
{
"command": "codeQLQueryHistory.showEvalLogViewer",
"when": "false"
},
{
"command": "codeQLQueryHistory.openQueryDirectory",
"when": "false"
@@ -978,6 +1022,10 @@
"command": "codeQLQueryHistory.openOnGithub",
"when": "false"
},
{
"command": "codeQLQueryHistory.copyRepoList",
"when": "false"
},
{
"command": "codeQLQueryHistory.showQueryText",
"when": "false"
@@ -1030,6 +1078,10 @@
"command": "codeQLAstViewer.clear",
"when": "false"
},
{
"command": "codeQLEvalLogViewer.clear",
"when": "false"
},
{
"command": "codeQLTests.acceptOutput",
"when": "false"
@@ -1071,6 +1123,10 @@
{
"command": "codeQL.previewQueryHelp",
"when": "resourceExtname == .qhelp && isWorkspaceTrusted"
},
{
"command": "codeQL.gotoQL",
"when": "editorLangId == ql-summary && config.codeQL.canary"
}
]
},
@@ -1096,6 +1152,11 @@
{
"id": "codeQLAstViewer",
"name": "AST Viewer"
},
{
"id": "codeQLEvalLogViewer",
"name": "Evaluator Log Viewer",
"when": "config.codeQL.canary"
}
]
},
@@ -1110,7 +1171,11 @@
},
{
"view": "codeQLDatabases",
"contents": "Add a CodeQL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From LGTM](command:codeQLDatabases.chooseDatabaseLgtm)"
"contents": "Add a CodeQL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From GitHub](command:codeQLDatabases.chooseDatabaseGithub)\n[From LGTM](command:codeQLDatabases.chooseDatabaseLgtm)"
},
{
"view": "codeQLEvalLogViewer",
"contents": "Run the 'Show Evaluator Log (UI)' command on a CodeQL query run in the Query History view."
}
]
},
@@ -1119,7 +1184,6 @@
"watch": "npm-run-all -p watch:*",
"watch:extension": "tsc --watch",
"watch:webpack": "gulp watchView",
"watch:css": "gulp watchCss",
"test": "mocha --exit -r ts-node/register test/pure-tests/**/*.ts",
"preintegration": "rm -rf ./out/vscode-tests && gulp",
"integration": "node ./out/vscode-tests/run-integration-tests.js no-workspace,minimal-workspace",
@@ -1130,16 +1194,20 @@
"format-staged": "lint-staged"
},
"dependencies": {
"@octokit/plugin-retry": "^3.0.9",
"@octokit/rest": "^18.5.6",
"@primer/octicons-react": "^16.3.0",
"@primer/react": "^35.0.0",
"@vscode/codicons": "^0.0.31",
"@vscode/webview-ui-toolkit": "^1.0.1",
"child-process-promise": "^2.2.1",
"classnames": "~2.2.6",
"d3": "^6.3.1",
"d3": "^7.6.1",
"d3-graphviz": "^2.6.1",
"fs-extra": "^10.0.1",
"glob-promise": "^3.4.0",
"js-yaml": "^3.14.0",
"glob-promise": "^4.2.2",
"immutable": "^4.0.0",
"js-yaml": "^4.1.0",
"minimist": "~1.2.6",
"nanoid": "^3.2.0",
"node-fetch": "~2.6.7",
@@ -1147,6 +1215,7 @@
"react": "^17.0.2",
"react-dom": "^17.0.2",
"semver": "~7.3.2",
"source-map": "^0.7.4",
"source-map-support": "^0.5.21",
"stream": "^0.0.2",
"stream-chain": "~2.2.4",
@@ -1168,7 +1237,7 @@
"@types/chai-as-promised": "~7.1.2",
"@types/child-process-promise": "^2.2.1",
"@types/classnames": "~2.2.9",
"@types/d3": "^6.2.0",
"@types/d3": "^7.4.0",
"@types/d3-graphviz": "^2.6.6",
"@types/del": "^4.0.0",
"@types/fs-extra": "^9.0.6",
@@ -1197,6 +1266,7 @@
"@types/unzipper": "~0.10.1",
"@types/vscode": "^1.59.0",
"@types/webpack": "^5.28.0",
"@types/webpack-env": "^1.18.0",
"@types/xml2js": "~0.4.4",
"@typescript-eslint/eslint-plugin": "^4.26.0",
"@typescript-eslint/parser": "^4.26.0",
@@ -1208,21 +1278,22 @@
"del": "^6.0.0",
"eslint": "~6.8.0",
"eslint-plugin-react": "~7.19.0",
"file-loader": "^6.2.0",
"glob": "^7.1.4",
"gulp": "^4.0.2",
"gulp-replace": "^1.1.3",
"gulp-sourcemaps": "^3.0.0",
"gulp-typescript": "^5.0.1",
"husky": "~4.2.5",
"husky": "~4.3.8",
"lint-staged": "~10.2.2",
"mocha": "^9.1.3",
"mini-css-extract-plugin": "^2.6.1",
"mocha": "^10.0.0",
"mocha-sinon": "~2.1.2",
"npm-run-all": "^4.1.5",
"prettier": "~2.0.5",
"proxyquire": "~2.1.3",
"sinon": "~13.0.1",
"sinon-chai": "~3.5.0",
"style-loader": "~0.23.1",
"through2": "^4.0.2",
"ts-loader": "^8.1.0",
"ts-node": "^10.7.0",
@@ -1231,7 +1302,7 @@
"typescript-formatter": "^7.2.2",
"vsce": "^2.7.0",
"vscode-test": "^1.4.0",
"webpack": "^5.28.0",
"webpack": "^5.62.2",
"webpack-cli": "^4.6.0"
},
"husky": {

View File

@@ -0,0 +1,118 @@
import {
WebviewPanel,
ExtensionContext,
window as Window,
ViewColumn,
Uri,
WebviewPanelOptions,
WebviewOptions
} from 'vscode';
import * as path from 'path';
import { DisposableObject } from './pure/disposable-object';
import { tmpDir } from './helpers';
import { getHtmlForWebview, WebviewMessage, WebviewView } from './interface-utils';
export type InterfacePanelConfig = {
viewId: string;
title: string;
viewColumn: ViewColumn;
view: WebviewView;
preserveFocus?: boolean;
additionalOptions?: WebviewPanelOptions & WebviewOptions;
}
export abstract class AbstractInterfaceManager<ToMessage extends WebviewMessage, FromMessage extends WebviewMessage> extends DisposableObject {
protected panel: WebviewPanel | undefined;
protected panelLoaded = false;
protected panelLoadedCallBacks: (() => void)[] = [];
constructor(
protected readonly ctx: ExtensionContext
) {
super();
}
protected get isShowingPanel() {
return !!this.panel;
}
protected getPanel(): WebviewPanel {
if (this.panel == undefined) {
const { ctx } = this;
const config = this.getPanelConfig();
this.panel = Window.createWebviewPanel(
config.viewId,
config.title,
{ viewColumn: ViewColumn.Active, preserveFocus: true },
{
enableScripts: true,
enableFindWidget: true,
retainContextWhenHidden: true,
...config.additionalOptions,
localResourceRoots: [
...(config.additionalOptions?.localResourceRoots ?? []),
Uri.file(tmpDir.name),
Uri.file(path.join(ctx.extensionPath, 'out'))
],
}
);
this.push(
this.panel.onDidDispose(
() => {
this.panel = undefined;
this.panelLoaded = false;
this.onPanelDispose();
},
null,
ctx.subscriptions
)
);
this.panel.webview.html = getHtmlForWebview(
ctx,
this.panel.webview,
config.view,
{
allowInlineStyles: true,
}
);
this.push(
this.panel.webview.onDidReceiveMessage(
async (e) => this.onMessage(e),
undefined,
ctx.subscriptions
)
);
}
return this.panel;
}
protected abstract getPanelConfig(): InterfacePanelConfig;
protected abstract onPanelDispose(): void;
protected abstract onMessage(msg: FromMessage): Promise<void>;
protected waitForPanelLoaded(): Promise<void> {
return new Promise((resolve) => {
if (this.panelLoaded) {
resolve();
} else {
this.panelLoadedCallBacks.push(resolve);
}
});
}
protected onWebViewLoaded(): void {
this.panelLoaded = true;
this.panelLoadedCallBacks.forEach((cb) => cb());
this.panelLoadedCallBacks = [];
}
protected postMessage(msg: ToMessage): Thenable<boolean> {
return this.getPanel().webview.postMessage(msg);
}
}

View File

@@ -167,21 +167,26 @@ type Archive = {
dirMap: DirectoryHierarchyMap;
};
async function parse_zip(zipPath: string): Promise<Archive> {
if (!await fs.pathExists(zipPath))
throw vscode.FileSystemError.FileNotFound(zipPath);
const archive: Archive = { unzipped: await unzipper.Open.file(zipPath), dirMap: new Map };
archive.unzipped.files.forEach(f => { ensureFile(archive.dirMap, path.resolve('/', f.path)); });
return archive;
}
export class ArchiveFileSystemProvider implements vscode.FileSystemProvider {
private readOnlyError = vscode.FileSystemError.NoPermissions('write operation attempted, but source archive filesystem is readonly');
private archives: Map<string, Archive> = new Map;
private archives: Map<string, Promise<Archive>> = new Map;
private async getArchive(zipPath: string): Promise<Archive> {
if (!this.archives.has(zipPath)) {
if (!await fs.pathExists(zipPath))
throw vscode.FileSystemError.FileNotFound(zipPath);
const archive: Archive = { unzipped: await unzipper.Open.file(zipPath), dirMap: new Map };
archive.unzipped.files.forEach(f => { ensureFile(archive.dirMap, path.resolve('/', f.path)); });
this.archives.set(zipPath, archive);
this.archives.set(zipPath, parse_zip(zipPath));
}
return this.archives.get(zipPath)!;
return await this.archives.get(zipPath)!;
}
root = new Directory('');
// metadata

View File

@@ -1,5 +1,6 @@
import * as vscode from 'vscode';
import * as Octokit from '@octokit/rest';
import { retry } from '@octokit/plugin-retry';
const GITHUB_AUTH_PROVIDER_ID = 'github';
@@ -35,12 +36,31 @@ export class Credentials {
return c;
}
private async createOctokit(createIfNone: boolean): Promise<Octokit.Octokit | undefined> {
/**
* Initializes an instance of credentials with an octokit instance using
* a token from the user's GitHub account. This method is meant to be
* used non-interactive environments such as tests.
*
* @param overrideToken The GitHub token to use for authentication.
* @returns An instance of credentials.
*/
static async initializeWithToken(overrideToken: string) {
const c = new Credentials();
c.octokit = await c.createOctokit(false, overrideToken);
return c;
}
private async createOctokit(createIfNone: boolean, overrideToken?: string): Promise<Octokit.Octokit | undefined> {
if (overrideToken) {
return new Octokit.Octokit({ auth: overrideToken, retry });
}
const session = await vscode.authentication.getSession(GITHUB_AUTH_PROVIDER_ID, SCOPES, { createIfNone });
if (session) {
return new Octokit.Octokit({
auth: session.accessToken
auth: session.accessToken,
retry
});
} else {
return undefined;
@@ -56,16 +76,27 @@ export class Credentials {
}));
}
async getOctokit(): Promise<Octokit.Octokit> {
/**
* Creates or returns an instance of Octokit.
*
* @param requireAuthentication Whether the Octokit instance needs to be authenticated as user.
* @returns An instance of Octokit.
*/
async getOctokit(requireAuthentication = true): Promise<Octokit.Octokit> {
if (this.octokit) {
return this.octokit;
}
this.octokit = await this.createOctokit(true);
// octokit shouldn't be undefined, since we've set "createIfNone: true".
// The following block is mainly here to prevent a compiler error.
this.octokit = await this.createOctokit(requireAuthentication);
if (!this.octokit) {
throw new Error('Did not initialize Octokit.');
if (requireAuthentication) {
throw new Error('Did not initialize Octokit.');
}
// We don't want to set this in this.octokit because that would prevent
// authenticating when requireCredentials is true.
return new Octokit.Octokit({ retry });
}
return this.octokit;
}

View File

@@ -168,7 +168,7 @@ export class CodeQLCliServer implements Disposable {
nullBuffer: Buffer;
/** Version of current cli, lazily computed by the `getVersion()` method */
private _version: SemVer | undefined;
private _version: Promise<SemVer> | undefined;
/**
* The languages supported by the current version of the CLI, computed by `getSupportedLanguages()`.
@@ -240,7 +240,7 @@ export class CodeQLCliServer implements Disposable {
/**
* Restart the server when the current command terminates
*/
private restartCliServer(): void {
restartCliServer(): void {
const callback = (): void => {
try {
this.killProcessIfRunning();
@@ -604,10 +604,14 @@ export class CodeQLCliServer implements Disposable {
}
/** Resolves the ML models that should be available when evaluating a query. */
async resolveMlModels(additionalPacks: string[]): Promise<MlModelsInfo> {
async resolveMlModels(additionalPacks: string[], queryPath: string): Promise<MlModelsInfo> {
const args = await this.cliConstraints.supportsPreciseResolveMlModels()
// use the dirname of the path so that we can handle query libraries
? [...this.getAdditionalPacksArg(additionalPacks), path.dirname(queryPath)]
: this.getAdditionalPacksArg(additionalPacks);
return await this.runJsonCodeQlCliCommand<MlModelsInfo>(
['resolve', 'ml-models'],
this.getAdditionalPacksArg(additionalPacks),
args,
'Resolving ML models',
false
);
@@ -679,12 +683,30 @@ export class CodeQLCliServer implements Disposable {
const subcommandArgs = [
'--format=text',
`--end-summary=${endSummaryPath}`,
...(await this.cliConstraints.supportsSourceMap() ? ['--sourcemap'] : []),
inputPath,
outputPath
];
return await this.runCodeQlCliCommand(['generate', 'log-summary'], subcommandArgs, 'Generating log summary');
}
/**
* Generate a JSON summary of an evaluation log.
* @param inputPath The path of an evaluation event log.
* @param outputPath The path to write a JSON summary of it to.
*/
async generateJsonLogSummary(
inputPath: string,
outputPath: string,
): Promise<string> {
const subcommandArgs = [
'--format=predicates',
inputPath,
outputPath
];
return await this.runCodeQlCliCommand(['generate', 'log-summary'], subcommandArgs, 'Generating JSON log summary');
}
/**
* Gets the results from a bqrs.
* @param bqrsPath The path to the bqrs.
@@ -914,8 +936,12 @@ export class CodeQLCliServer implements Disposable {
return this.runJsonCodeQlCliCommand(['pack', 'download'], packs, 'Downloading packs');
}
async packInstall(dir: string) {
return this.runJsonCodeQlCliCommand(['pack', 'install'], [dir], 'Installing pack dependencies');
async packInstall(dir: string, forceUpdate = false) {
const args = [dir];
if (forceUpdate) {
args.push('--mode', 'update');
}
return this.runJsonCodeQlCliCommand(['pack', 'install'], args, 'Installing pack dependencies');
}
async packBundle(dir: string, workspaceFolders: string[], outputPath: string, precompile = true): Promise<void> {
@@ -959,13 +985,13 @@ export class CodeQLCliServer implements Disposable {
public async getVersion() {
if (!this._version) {
this._version = await this.refreshVersion();
this._version = this.refreshVersion();
// this._version is only undefined upon config change, so we reset CLI-based context key only when necessary.
await commands.executeCommand(
'setContext', 'codeql.supportsEvalLog', await this.cliConstraints.supportsPerQueryEvalLog()
);
}
return this._version;
return await this._version;
}
private async refreshVersion() {
@@ -1264,6 +1290,11 @@ export class CliVersionConstraint {
*/
public static CLI_VERSION_WITH_RESOLVE_ML_MODELS = new SemVer('2.7.3');
/**
* CLI version where the `resolve ml-models` subcommand was enhanced to work with packaging.
*/
public static CLI_VERSION_WITH_PRECISE_RESOLVE_ML_MODELS = new SemVer('2.10.0');
/**
* CLI version where the `--old-eval-stats` option to the query server was introduced.
*/
@@ -1291,6 +1322,11 @@ export class CliVersionConstraint {
*/
public static CLI_VERSION_WITH_PER_QUERY_EVAL_LOG = new SemVer('2.9.0');
/**
* CLI version that supports the `--sourcemap` option for log generation.
*/
public static CLI_VERSION_WITH_SOURCEMAP = new SemVer('2.10.3');
constructor(private readonly cli: CodeQLCliServer) {
/**/
}
@@ -1339,6 +1375,10 @@ export class CliVersionConstraint {
return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_RESOLVE_ML_MODELS);
}
async supportsPreciseResolveMlModels() {
return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_PRECISE_RESOLVE_ML_MODELS);
}
async supportsOldEvalStats() {
return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_OLD_EVAL_STATS);
}
@@ -1354,4 +1394,8 @@ export class CliVersionConstraint {
async supportsPerQueryEvalLog() {
return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG);
}
async supportsSourceMap() {
return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_SOURCEMAP);
}
}

View File

@@ -1,14 +1,8 @@
import { DisposableObject } from '../pure/disposable-object';
import {
WebviewPanel,
ExtensionContext,
window as Window,
ViewColumn,
Uri,
} from 'vscode';
import * as path from 'path';
import { tmpDir } from '../helpers';
import {
FromCompareViewMessage,
ToCompareViewMessage,
@@ -17,26 +11,24 @@ import {
import { Logger } from '../logging';
import { CodeQLCliServer } from '../cli';
import { DatabaseManager } from '../databases';
import { getHtmlForWebview, jumpToLocation } from '../interface-utils';
import { jumpToLocation } from '../interface-utils';
import { transformBqrsResultSet, RawResultSet, BQRSInfo } from '../pure/bqrs-cli-types';
import resultsDiff from './resultsDiff';
import { CompletedLocalQueryInfo } from '../query-results';
import { getErrorMessage } from '../pure/helpers-pure';
import { HistoryItemLabelProvider } from '../history-item-label-provider';
import { AbstractInterfaceManager, InterfacePanelConfig } from '../abstract-interface-manager';
interface ComparePair {
from: CompletedLocalQueryInfo;
to: CompletedLocalQueryInfo;
}
export class CompareInterfaceManager extends DisposableObject {
export class CompareInterfaceManager extends AbstractInterfaceManager<ToCompareViewMessage, FromCompareViewMessage> {
private comparePair: ComparePair | undefined;
private panel: WebviewPanel | undefined;
private panelLoaded = false;
private panelLoadedCallBacks: (() => void)[] = [];
constructor(
private ctx: ExtensionContext,
ctx: ExtensionContext,
private databaseManager: DatabaseManager,
private cliServer: CodeQLCliServer,
private logger: Logger,
@@ -45,7 +37,7 @@ export class CompareInterfaceManager extends DisposableObject {
item: CompletedLocalQueryInfo
) => Promise<void>
) {
super();
super(ctx);
}
async showResults(
@@ -103,73 +95,24 @@ export class CompareInterfaceManager extends DisposableObject {
}
}
getPanel(): WebviewPanel {
if (this.panel == undefined) {
const { ctx } = this;
const panel = (this.panel = Window.createWebviewPanel(
'compareView',
'Compare CodeQL Query Results',
{ viewColumn: ViewColumn.Active, preserveFocus: true },
{
enableScripts: true,
enableFindWidget: true,
retainContextWhenHidden: true,
localResourceRoots: [
Uri.file(tmpDir.name),
Uri.file(path.join(this.ctx.extensionPath, 'out')),
],
}
));
this.push(this.panel.onDidDispose(
() => {
this.panel = undefined;
this.comparePair = undefined;
},
null,
ctx.subscriptions
));
const scriptPathOnDisk = Uri.file(
ctx.asAbsolutePath('out/compareView.js')
);
const stylesheetPathOnDisk = Uri.file(
ctx.asAbsolutePath('out/view/resultsView.css')
);
panel.webview.html = getHtmlForWebview(
panel.webview,
scriptPathOnDisk,
[stylesheetPathOnDisk],
false
);
this.push(panel.webview.onDidReceiveMessage(
async (e) => this.handleMsgFromView(e),
undefined,
ctx.subscriptions
));
}
return this.panel;
protected getPanelConfig(): InterfacePanelConfig {
return {
viewId: 'compareView',
title: 'Compare CodeQL Query Results',
viewColumn: ViewColumn.Active,
preserveFocus: true,
view: 'compare',
};
}
private waitForPanelLoaded(): Promise<void> {
return new Promise((resolve) => {
if (this.panelLoaded) {
resolve();
} else {
this.panelLoadedCallBacks.push(resolve);
}
});
protected onPanelDispose(): void {
this.comparePair = undefined;
}
private async handleMsgFromView(
msg: FromCompareViewMessage
): Promise<void> {
protected async onMessage(msg: FromCompareViewMessage): Promise<void> {
switch (msg.t) {
case 'compareViewLoaded':
this.panelLoaded = true;
this.panelLoadedCallBacks.forEach((cb) => cb());
this.panelLoadedCallBacks = [];
this.onWebViewLoaded();
break;
case 'changeCompare':
@@ -186,10 +129,6 @@ export class CompareInterfaceManager extends DisposableObject {
}
}
private postMessage(msg: ToCompareViewMessage): Thenable<boolean> {
return this.getPanel().webview.postMessage(msg);
}
private async findCommonResultSetNames(
from: CompletedLocalQueryInfo,
to: CompletedLocalQueryInfo,

View File

@@ -1,13 +0,0 @@
module.exports = {
env: {
browser: true
},
extends: [
"plugin:react/recommended"
],
settings: {
react: {
version: 'detect'
}
}
}

View File

@@ -1,18 +0,0 @@
{
"compilerOptions": {
"module": "esnext",
"moduleResolution": "node",
"target": "es6",
"outDir": "out",
"lib": ["ES2021", "dom"],
"jsx": "react",
"sourceMap": true,
"rootDir": "..",
"strict": true,
"noUnusedLocals": true,
"noImplicitReturns": true,
"noFallthroughCasesInSwitch": true,
"experimentalDecorators": true
},
"exclude": ["node_modules"]
}

View File

@@ -2,7 +2,7 @@ import { DisposableObject } from './pure/disposable-object';
import { workspace, Event, EventEmitter, ConfigurationChangeEvent, ConfigurationTarget } from 'vscode';
import { DistributionManager } from './distribution';
import { logger } from './logging';
import { ONE_DAY_IN_MS } from './pure/helpers-pure';
import { ONE_DAY_IN_MS } from './pure/time';
/** Helper class to look up a labelled (and possibly nested) setting. */
export class Setting {
@@ -59,7 +59,7 @@ const PERSONAL_ACCESS_TOKEN_SETTING = new Setting('personalAccessToken', DISTRIB
// Query History configuration
const QUERY_HISTORY_SETTING = new Setting('queryHistory', ROOT_SETTING);
const QUERY_HISTORY_FORMAT_SETTING = new Setting('format', QUERY_HISTORY_SETTING);
const QUERY_HISTORY_TTL = new Setting('format', QUERY_HISTORY_SETTING);
const QUERY_HISTORY_TTL = new Setting('ttl', QUERY_HISTORY_SETTING);
/** When these settings change, the distribution should be updated. */
const DISTRIBUTION_CHANGE_SETTINGS = [CUSTOM_CODEQL_PATH_SETTING, INCLUDE_PRERELEASE_SETTING, PERSONAL_ACCESS_TOKEN_SETTING];
@@ -342,6 +342,21 @@ export async function setRemoteRepositoryLists(lists: Record<string, string[]> |
await REMOTE_REPO_LISTS.updateValue(lists, ConfigurationTarget.Global);
}
/**
* Path to a file that contains lists of GitHub repositories that you want to query remotely via
* the "Run Variant Analysis" command.
* Note: This command is only available for internal users.
*
* This setting should be a path to a JSON file that contains a JSON object where each key is a
* user-specified name (string), and the value is an array of GitHub repositories
* (of the form `<owner>/<repo>`).
*/
const REPO_LISTS_PATH = new Setting('repositoryListsPath', REMOTE_QUERIES_SETTING);
export function getRemoteRepositoryListsPath(): string | undefined {
return REPO_LISTS_PATH.getValue<string>() || undefined;
}
/**
* The name of the "controller" repository that you want to use with the "Run Variant Analysis" command.
* Note: This command is only available for internal users.

View File

@@ -45,7 +45,7 @@ async function resolveQueriesFromPacks(cli: CodeQLCliServer, qlpacks: string[],
}
});
}
await fs.writeFile(suiteFile, yaml.safeDump(suiteYaml), 'utf8');
await fs.writeFile(suiteFile, yaml.dump(suiteYaml), 'utf8');
const queries = await cli.resolveQueriesInSuite(suiteFile, helpers.getOnDiskWorkspaceFolders());
return queries;

View File

@@ -10,6 +10,8 @@ import {
import { CodeQLCliServer } from './cli';
import * as fs from 'fs-extra';
import * as path from 'path';
import * as Octokit from '@octokit/rest';
import { retry } from '@octokit/plugin-retry';
import { DatabaseManager, DatabaseItem } from './databases';
import {
@@ -76,7 +78,7 @@ export async function promptImportInternetDatabase(
export async function promptImportGithubDatabase(
databaseManager: DatabaseManager,
storagePath: string,
credentials: Credentials,
credentials: Credentials | undefined,
progress: ProgressCallback,
token: CancellationToken,
cli?: CodeQLCliServer
@@ -99,14 +101,15 @@ export async function promptImportGithubDatabase(
throw new Error(`Invalid GitHub repository: ${githubRepo}`);
}
const result = await convertGithubNwoToDatabaseUrl(githubRepo, credentials, progress);
const octokit = credentials ? await credentials.getOctokit(true) : new Octokit.Octokit({ retry });
const result = await convertGithubNwoToDatabaseUrl(githubRepo, octokit, progress);
if (!result) {
return;
}
const { databaseUrl, name, owner } = result;
const octokit = await credentials.getOctokit();
/**
* The 'token' property of the token object returned by `octokit.auth()`.
* The object is undocumented, but looks something like this:
@@ -118,14 +121,9 @@ export async function promptImportGithubDatabase(
* We only need the actual token string.
*/
const octokitToken = (await octokit.auth() as { token: string })?.token;
if (!octokitToken) {
// Just print a generic error message for now. Ideally we could show more debugging info, like the
// octokit object, but that would expose a user token.
throw new Error('Unable to get GitHub token.');
}
const item = await databaseArchiveFetcher(
databaseUrl,
{ 'Accept': 'application/zip', 'Authorization': `Bearer ${octokitToken}` },
{ 'Accept': 'application/zip', 'Authorization': octokitToken ? `Bearer ${octokitToken}` : '' },
databaseManager,
storagePath,
`${owner}/${name}`,
@@ -523,7 +521,7 @@ function convertGitHubUrlToNwo(githubUrl: string): string | undefined {
export async function convertGithubNwoToDatabaseUrl(
githubRepo: string,
credentials: Credentials,
octokit: Octokit.Octokit,
progress: ProgressCallback): Promise<{
databaseUrl: string,
owner: string,
@@ -533,7 +531,6 @@ export async function convertGithubNwoToDatabaseUrl(
const nwo = convertGitHubUrlToNwo(githubRepo) || githubRepo;
const [owner, repo] = nwo.split('/');
const octokit = await credentials.getOctokit();
const response = await octokit.request('GET /repos/:owner/:repo/code-scanning/codeql/databases', { owner, repo });
const languages = response.data.map((db: any) => db.language);

View File

@@ -40,6 +40,7 @@ import {
import { CancellationToken } from 'vscode';
import { asyncFilter, getErrorMessage } from './pure/helpers-pure';
import { Credentials } from './authentication';
import { isCanary } from './config';
type ThemableIconPath = { light: string; dark: string } | string;
@@ -301,7 +302,7 @@ export class DatabaseUI extends DisposableObject {
progress: ProgressCallback,
token: CancellationToken
) => {
const credentials = await this.getCredentials();
const credentials = isCanary() ? await this.getCredentials() : undefined;
await this.handleChooseDatabaseGithub(credentials, progress, token);
},
{
@@ -480,7 +481,7 @@ export class DatabaseUI extends DisposableObject {
};
handleChooseDatabaseGithub = async (
credentials: Credentials,
credentials: Credentials | undefined,
progress: ProgressCallback,
token: CancellationToken
): Promise<DatabaseItem | undefined> => {

View File

@@ -0,0 +1,67 @@
import { ChildEvalLogTreeItem, EvalLogTreeItem } from './eval-log-viewer';
import { EvalLogData as EvalLogData } from './pure/log-summary-parser';
/** Builds the tree data for the evaluator log viewer for a single query run. */
export default class EvalLogTreeBuilder {
private queryName: string;
private evalLogDataItems: EvalLogData[];
constructor(queryName: string, evaluatorLogDataItems: EvalLogData[]) {
this.queryName = queryName;
this.evalLogDataItems = evaluatorLogDataItems;
}
async getRoots(): Promise<EvalLogTreeItem[]> {
return await this.parseRoots();
}
private async parseRoots(): Promise<EvalLogTreeItem[]> {
const roots: EvalLogTreeItem[] = [];
// Once the viewer can show logs for multiple queries, there will be more than 1 item at the root
// level. For now, there will always be one root (the one query being shown).
const queryItem: EvalLogTreeItem = {
label: this.queryName,
children: [] // Will assign predicate items as children shortly.
};
// Display descriptive message when no data exists
if (this.evalLogDataItems.length === 0) {
const noResultsItem: ChildEvalLogTreeItem = {
label: 'No predicates evaluated in this query run.',
parent: queryItem,
children: [],
};
queryItem.children.push(noResultsItem);
}
// For each predicate, create a TreeItem object with appropriate parents/children
this.evalLogDataItems.forEach(logDataItem => {
const predicateLabel = `${logDataItem.predicateName} (${logDataItem.resultSize} tuples, ${logDataItem.millis} ms)`;
const predicateItem: ChildEvalLogTreeItem = {
label: predicateLabel,
parent: queryItem,
children: [] // Will assign pipeline items as children shortly.
};
for (const [pipelineName, steps] of Object.entries(logDataItem.ra)) {
const pipelineLabel = `Pipeline: ${pipelineName}`;
const pipelineItem: ChildEvalLogTreeItem = {
label: pipelineLabel,
parent: predicateItem,
children: [] // Will assign step items as children shortly.
};
predicateItem.children.push(pipelineItem);
pipelineItem.children = steps.map((step: string) => ({
label: step,
parent: pipelineItem,
children: []
}));
}
queryItem.children.push(predicateItem);
});
roots.push(queryItem);
return roots;
}
}

View File

@@ -0,0 +1,92 @@
import { window, TreeDataProvider, TreeView, TreeItem, ProviderResult, Event, EventEmitter, TreeItemCollapsibleState } from 'vscode';
import { commandRunner } from './commandRunner';
import { DisposableObject } from './pure/disposable-object';
import { showAndLogErrorMessage } from './helpers';
export interface EvalLogTreeItem {
label?: string;
children: ChildEvalLogTreeItem[];
}
export interface ChildEvalLogTreeItem extends EvalLogTreeItem {
parent: ChildEvalLogTreeItem | EvalLogTreeItem;
}
/** Provides data from parsed CodeQL evaluator logs to be rendered in a tree view. */
class EvalLogDataProvider extends DisposableObject implements TreeDataProvider<EvalLogTreeItem> {
public roots: EvalLogTreeItem[] = [];
private _onDidChangeTreeData: EventEmitter<EvalLogTreeItem | undefined | null | void> = new EventEmitter<EvalLogTreeItem | undefined | null | void>();
readonly onDidChangeTreeData: Event<EvalLogTreeItem | undefined | null | void> = this._onDidChangeTreeData.event;
refresh(): void {
this._onDidChangeTreeData.fire();
}
getTreeItem(element: EvalLogTreeItem): TreeItem | Thenable<TreeItem> {
const state = element.children.length
? TreeItemCollapsibleState.Collapsed
: TreeItemCollapsibleState.None;
const treeItem = new TreeItem(element.label || '', state);
treeItem.tooltip = `${treeItem.label} || ''}`;
return treeItem;
}
getChildren(element?: EvalLogTreeItem): ProviderResult<EvalLogTreeItem[]> {
// If no item is passed, return the root.
if (!element) {
return this.roots || [];
}
// Otherwise it is called with an existing item, to load its children.
return element.children;
}
getParent(element: ChildEvalLogTreeItem): ProviderResult<EvalLogTreeItem> {
return element.parent;
}
}
/** Manages a tree viewer of structured evaluator logs. */
export class EvalLogViewer extends DisposableObject {
private treeView: TreeView<EvalLogTreeItem>;
private treeDataProvider: EvalLogDataProvider;
constructor() {
super();
this.treeDataProvider = new EvalLogDataProvider();
this.treeView = window.createTreeView('codeQLEvalLogViewer', {
treeDataProvider: this.treeDataProvider,
showCollapseAll: true
});
this.push(this.treeView);
this.push(this.treeDataProvider);
this.push(
commandRunner('codeQLEvalLogViewer.clear', async () => {
this.clear();
})
);
}
private clear(): void {
this.treeDataProvider.roots = [];
this.treeDataProvider.refresh();
this.treeView.message = undefined;
}
// Called when the Show Evaluator Log (UI) command is run on a new query.
updateRoots(roots: EvalLogTreeItem[]): void {
this.treeDataProvider.roots = roots;
this.treeDataProvider.refresh();
this.treeView.message = 'Viewer for query run:'; // Currently only one query supported at a time.
// Handle error on reveal. This could happen if
// the tree view is disposed during the reveal.
this.treeView.reveal(roots[0], { focus: false })?.then(
() => { /**/ },
err => showAndLogErrorMessage(err)
);
}
}

View File

@@ -95,9 +95,13 @@ import { RemoteQueriesManager } from './remote-queries/remote-queries-manager';
import { RemoteQueryResult } from './remote-queries/remote-query-result';
import { URLSearchParams } from 'url';
import { handleDownloadPacks, handleInstallPackDependencies } from './packaging';
import { RemoteQueryHistoryItem } from './remote-queries/remote-query-history-item';
import { HistoryItemLabelProvider } from './history-item-label-provider';
import { exportRemoteQueryResults } from './remote-queries/export-results';
import { RemoteQuery } from './remote-queries/remote-query';
import { EvalLogViewer } from './eval-log-viewer';
import { SummaryLanguageSupport } from './log-insights/summary-language-support';
import { JoinOrderScannerProvider } from './log-insights/join-order';
import { LogScannerService } from './log-insights/log-scanner-service';
/**
* extension.ts
@@ -442,6 +446,10 @@ async function activateWithInstalledDistribution(
databaseUI.init();
ctx.subscriptions.push(databaseUI);
void logger.log('Initializing evaluator log viewer.');
const evalLogViewer = new EvalLogViewer();
ctx.subscriptions.push(evalLogViewer);
void logger.log('Initializing query history manager.');
const queryHistoryConfigurationListener = new QueryHistoryConfigListener();
ctx.subscriptions.push(queryHistoryConfigurationListener);
@@ -451,10 +459,21 @@ async function activateWithInstalledDistribution(
await fs.ensureDir(queryStorageDir);
const labelProvider = new HistoryItemLabelProvider(queryHistoryConfigurationListener);
void logger.log('Initializing results panel interface.');
const intm = new InterfaceManager(ctx, dbm, cliServer, queryServerLogger, labelProvider);
ctx.subscriptions.push(intm);
void logger.log('Initializing variant analysis manager.');
const rqm = new RemoteQueriesManager(ctx, cliServer, queryStorageDir, logger);
ctx.subscriptions.push(rqm);
void logger.log('Initializing query history.');
const qhm = new QueryHistoryManager(
qs,
dbm,
intm,
rqm,
evalLogViewer,
queryStorageDir,
ctx,
queryHistoryConfigurationListener,
@@ -463,17 +482,16 @@ async function activateWithInstalledDistribution(
showResultsForComparison(from, to),
);
qhm.onWillOpenQueryItem(async item => {
if (item.t === 'local' && item.completed) {
await showResultsForCompletedQuery(item as CompletedLocalQueryInfo, WebviewReveal.Forced);
}
});
ctx.subscriptions.push(qhm);
void logger.log('Initializing results panel interface.');
const intm = new InterfaceManager(ctx, dbm, cliServer, queryServerLogger, labelProvider);
ctx.subscriptions.push(intm);
void logger.log('Initializing evaluation log scanners.');
const logScannerService = new LogScannerService(qhm);
ctx.subscriptions.push(logScannerService);
ctx.subscriptions.push(logScannerService.scanners.registerLogScannerProvider(new JoinOrderScannerProvider()));
void logger.log('Reading query history');
await qhm.readQueryHistory();
void logger.log('Initializing compare panel interface.');
const cmpm = new CompareInterfaceManager(
@@ -545,7 +563,7 @@ async function activateWithInstalledDistribution(
undefined,
item,
);
item.completeThisQuery(completedQueryInfo);
qhm.completeQuery(item, completedQueryInfo);
await showResultsForCompletedQuery(item as CompletedLocalQueryInfo, WebviewReveal.NotForced);
// Note we must update the query history view after showing results as the
// display and sorting might depend on the number of results
@@ -844,14 +862,6 @@ async function activateWithInstalledDistribution(
)
);
void logger.log('Initializing variant analysis results view.');
const rqm = new RemoteQueriesManager(ctx, cliServer, qhm, queryStorageDir, logger);
ctx.subscriptions.push(rqm);
// wait until after the remote queries manager is initialized to read the query history
// since the rqm is notified of queries being added.
await qhm.readQueryHistory();
registerRemoteQueryTextProvider();
@@ -884,11 +894,18 @@ async function activateWithInstalledDistribution(
ctx.subscriptions.push(
commandRunner('codeQL.monitorRemoteQuery', async (
queryItem: RemoteQueryHistoryItem,
queryId: string,
query: RemoteQuery,
token: CancellationToken) => {
await rqm.monitorRemoteQuery(queryItem, token);
await rqm.monitorRemoteQuery(queryId, query, token);
}));
ctx.subscriptions.push(
commandRunner('codeQL.copyRepoList', async (queryId: string) => {
await rqm.copyRemoteQueryRepoListToClipboard(queryId);
})
);
ctx.subscriptions.push(
commandRunner('codeQL.autoDownloadRemoteQueryResults', async (
queryResult: RemoteQueryResult,
@@ -921,6 +938,8 @@ async function activateWithInstalledDistribution(
progress: ProgressCallback,
token: CancellationToken
) => {
// We restart the CLI server too, to ensure they are the same version
cliServer.restartCliServer();
await qs.restartQueryServer(progress, token);
void showAndLogInformationMessage('CodeQL Query Server restarted.', {
outputLogger: queryServerLogger,
@@ -953,7 +972,7 @@ async function activateWithInstalledDistribution(
progress: ProgressCallback,
token: CancellationToken
) => {
const credentials = await Credentials.initialize(ctx);
const credentials = isCanary() ? await Credentials.initialize(ctx) : undefined;
await databaseUI.handleChooseDatabaseGithub(credentials, progress, token);
},
{
@@ -1001,19 +1020,16 @@ async function activateWithInstalledDistribution(
}
};
// The "authenticateToGitHub" command is internal-only.
ctx.subscriptions.push(
commandRunner('codeQL.authenticateToGitHub', async () => {
if (isCanary()) {
/**
* Credentials for authenticating to GitHub.
* These are used when making API calls.
*/
const credentials = await Credentials.initialize(ctx);
const octokit = await credentials.getOctokit();
const userInfo = await octokit.users.getAuthenticated();
void showAndLogInformationMessage(`Authenticated to GitHub as user: ${userInfo.data.login}`);
}
/**
* Credentials for authenticating to GitHub.
* These are used when making API calls.
*/
const credentials = await Credentials.initialize(ctx);
const octokit = await credentials.getOctokit();
const userInfo = await octokit.users.getAuthenticated();
void showAndLogInformationMessage(`Authenticated to GitHub as user: ${userInfo.data.login}`);
}));
ctx.subscriptions.push(
@@ -1042,6 +1058,8 @@ async function activateWithInstalledDistribution(
})
);
ctx.subscriptions.push(new SummaryLanguageSupport());
void logger.log('Starting language server.');
ctx.subscriptions.push(client.start());

View File

@@ -289,7 +289,7 @@ interface QlPackWithPath {
async function findDbschemePack(packs: QlPackWithPath[], dbschemePath: string): Promise<{ name: string; isLibraryPack: boolean; }> {
for (const { packDir, packName } of packs) {
if (packDir !== undefined) {
const qlpack = yaml.safeLoad(await fs.readFile(path.join(packDir, 'qlpack.yml'), 'utf8')) as { dbscheme?: string; library?: boolean; };
const qlpack = yaml.load(await fs.readFile(path.join(packDir, 'qlpack.yml'), 'utf8')) as { dbscheme?: string; library?: boolean; };
if (qlpack.dbscheme !== undefined && path.basename(qlpack.dbscheme) === path.basename(dbschemePath)) {
return {
name: packName,
@@ -581,3 +581,11 @@ export async function* walkDirectory(dir: string): AsyncIterableIterator<string>
}
}
}
/**
* Pluralizes a word.
* Example: Returns "N repository" if N is one, "N repositories" otherwise.
*/
export function pluralize(numItems: number | undefined, singular: string, plural: string): string {
return numItems ? `${numItems} ${numItems === 1 ? singular : plural}` : '';
}

View File

@@ -3,6 +3,7 @@ import * as path from 'path';
import { QueryHistoryConfig } from './config';
import { LocalQueryInfo, QueryHistoryInfo } from './query-results';
import { RemoteQueryHistoryItem } from './remote-queries/remote-query-history-item';
import { pluralize } from './helpers';
interface InterpolateReplacements {
t: string; // Start time
@@ -45,10 +46,12 @@ export class HistoryItemLabelProvider {
private interpolate(rawLabel: string, replacements: InterpolateReplacements): string {
return rawLabel.replace(/%(.)/g, (match, key: keyof InterpolateReplacements) => {
const label = rawLabel.replace(/%(.)/g, (match, key: keyof InterpolateReplacements) => {
const replacement = replacements[key];
return replacement !== undefined ? replacement : match;
});
return label.replace(/\s+/g, ' ');
}
private getLocalInterpolateReplacements(item: LocalQueryInfo): InterpolateReplacements {
@@ -57,23 +60,31 @@ export class HistoryItemLabelProvider {
t: item.startTime,
q: item.getQueryName(),
d: item.initialInfo.databaseInfo.name,
r: `${resultCount} results`,
r: `(${resultCount} results)`,
s: statusString,
f: item.getQueryFileName(),
'%': '%',
};
}
// Return the number of repositories queried if available. Otherwise, use the controller repository name.
private buildRepoLabel(item: RemoteQueryHistoryItem): string {
const repositoryCount = item.remoteQuery.repositoryCount;
if (repositoryCount) {
return pluralize(repositoryCount, 'repository', 'repositories');
}
return `${item.remoteQuery.controllerRepository.owner}/${item.remoteQuery.controllerRepository.name}`;
}
private getRemoteInterpolateReplacements(item: RemoteQueryHistoryItem): InterpolateReplacements {
const resultCount = item.resultCount ? `(${pluralize(item.resultCount, 'result', 'results')})` : '';
return {
t: new Date(item.remoteQuery.executionStartTime).toLocaleString(env.language),
q: item.remoteQuery.queryName,
// There is no database name for remote queries. Instead use the controller repository name.
d: `${item.remoteQuery.controllerRepository.owner}/${item.remoteQuery.controllerRepository.name}`,
// There is no synchronous way to get the results count.
r: '',
q: `${item.remoteQuery.queryName} (${item.remoteQuery.language})`,
d: this.buildRepoLabel(item),
r: resultCount,
s: item.status,
f: path.basename(item.remoteQuery.queryFilePath),
'%': '%'

View File

@@ -4,6 +4,7 @@ import {
Uri,
Location,
Range,
ExtensionContext,
WebviewPanel,
Webview,
workspace,
@@ -111,16 +112,36 @@ export function tryResolveLocation(
}
}
export type WebviewView = 'results' | 'compare' | 'remote-queries';
export interface WebviewMessage {
t: string;
}
/**
* Returns HTML to populate the given webview.
* Uses a content security policy that only loads the given script.
*/
export function getHtmlForWebview(
ctx: ExtensionContext,
webview: Webview,
scriptUriOnDisk: Uri,
stylesheetUrisOnDisk: Uri[],
allowInlineStyles: boolean
view: WebviewView,
{
allowInlineStyles,
}: {
allowInlineStyles?: boolean;
} = {
allowInlineStyles: false,
}
): string {
const scriptUriOnDisk = Uri.file(
ctx.asAbsolutePath('out/webview.js')
);
const stylesheetUrisOnDisk = [
Uri.file(ctx.asAbsolutePath('out/webview.css'))
];
// Convert the on-disk URIs into webview URIs.
const scriptWebviewUri = webview.asWebviewUri(scriptUriOnDisk);
const stylesheetWebviewUris = stylesheetUrisOnDisk.map(stylesheetUriOnDisk =>
@@ -137,6 +158,8 @@ export function getHtmlForWebview(
? `${webview.cspSource} vscode-file: 'unsafe-inline'`
: `'nonce-${nonce}'`;
const fontSrc = webview.cspSource;
/*
* Content security policy:
* default-src: allow nothing by default.
@@ -149,11 +172,11 @@ export function getHtmlForWebview(
<html>
<head>
<meta http-equiv="Content-Security-Policy"
content="default-src 'none'; script-src 'nonce-${nonce}'; style-src ${styleSrc}; connect-src ${webview.cspSource};">
content="default-src 'none'; script-src 'nonce-${nonce}'; font-src ${fontSrc}; style-src ${styleSrc}; connect-src ${webview.cspSource};">
${stylesheetsHtmlLines.join(` ${os.EOL}`)}
</head>
<body>
<div id=root>
<div id=root data-view="${view}">
</div>
<script nonce="${nonce}" src="${scriptWebviewUri}">
</script>

View File

@@ -1,6 +1,4 @@
import * as path from 'path';
import * as Sarif from 'sarif';
import { DisposableObject } from './pure/disposable-object';
import * as vscode from 'vscode';
import {
Diagnostic,
@@ -14,7 +12,7 @@ import {
import * as cli from './cli';
import { CodeQLCliServer } from './cli';
import { DatabaseEventKind, DatabaseItem, DatabaseManager } from './databases';
import { showAndLogErrorMessage, tmpDir } from './helpers';
import { showAndLogErrorMessage } from './helpers';
import { assertNever, getErrorMessage, getErrorStack } from './pure/helpers-pure';
import {
FromResultsViewMsg,
@@ -40,13 +38,13 @@ import {
WebviewReveal,
fileUriToWebviewUri,
tryResolveLocation,
getHtmlForWebview,
shownLocationDecoration,
shownLocationLineDecoration,
jumpToLocation,
} from './interface-utils';
import { getDefaultResultSetName, ParsedResultSets } from './pure/interface-types';
import { RawResultSet, transformBqrsResultSet, ResultSetSchema } from './pure/bqrs-cli-types';
import { AbstractInterfaceManager, InterfacePanelConfig } from './abstract-interface-manager';
import { PAGE_SIZE } from './config';
import { CompletedLocalQueryInfo } from './query-results';
import { HistoryItemLabelProvider } from './history-item-label-provider';
@@ -122,12 +120,9 @@ function numInterpretedPages(interpretation: Interpretation | undefined): number
return Math.ceil(n / pageSize);
}
export class InterfaceManager extends DisposableObject {
export class InterfaceManager extends AbstractInterfaceManager<IntoResultsViewMsg, FromResultsViewMsg> {
private _displayedQuery?: CompletedLocalQueryInfo;
private _interpretation?: Interpretation;
private _panel: vscode.WebviewPanel | undefined;
private _panelLoaded = false;
private _panelLoadedCallBacks: (() => void)[] = [];
private readonly _diagnosticCollection = languages.createDiagnosticCollection(
'codeql-query-results'
@@ -140,7 +135,7 @@ export class InterfaceManager extends DisposableObject {
public logger: Logger,
private labelProvider: HistoryItemLabelProvider
) {
super();
super(ctx);
this.push(this._diagnosticCollection);
this.push(
vscode.window.onDidChangeTextEditorSelection(
@@ -165,7 +160,7 @@ export class InterfaceManager extends DisposableObject {
this.databaseManager.onDidChangeDatabaseItem(({ kind }) => {
if (kind === DatabaseEventKind.Remove) {
this._diagnosticCollection.clear();
if (this.isShowingPanel()) {
if (this.isShowingPanel) {
void this.postMessage({
t: 'untoggleShowProblems'
});
@@ -179,59 +174,81 @@ export class InterfaceManager extends DisposableObject {
await this.postMessage({ t: 'navigatePath', direction });
}
private isShowingPanel() {
return !!this._panel;
protected getPanelConfig(): InterfacePanelConfig {
return {
viewId: 'resultsView',
title: 'CodeQL Query Results',
viewColumn: this.chooseColumnForWebview(),
preserveFocus: true,
view: 'results',
};
}
// Returns the webview panel, creating it if it doesn't already
// exist.
getPanel(): vscode.WebviewPanel {
if (this._panel == undefined) {
const { ctx } = this;
const webViewColumn = this.chooseColumnForWebview();
const panel = (this._panel = Window.createWebviewPanel(
'resultsView', // internal name
'CodeQL Query Results', // user-visible name
{ viewColumn: webViewColumn, preserveFocus: true },
{
enableScripts: true,
enableFindWidget: true,
retainContextWhenHidden: true,
localResourceRoots: [
vscode.Uri.file(tmpDir.name),
vscode.Uri.file(path.join(this.ctx.extensionPath, 'out'))
]
}
));
protected onPanelDispose(): void {
this._displayedQuery = undefined;
}
this.push(this._panel.onDidDispose(
() => {
this._panel = undefined;
this._displayedQuery = undefined;
this._panelLoaded = false;
},
null,
ctx.subscriptions
));
const scriptPathOnDisk = vscode.Uri.file(
ctx.asAbsolutePath('out/resultsView.js')
);
const stylesheetPathOnDisk = vscode.Uri.file(
ctx.asAbsolutePath('out/view/resultsView.css')
);
panel.webview.html = getHtmlForWebview(
panel.webview,
scriptPathOnDisk,
[stylesheetPathOnDisk],
false
);
this.push(panel.webview.onDidReceiveMessage(
async (e) => this.handleMsgFromView(e),
undefined,
ctx.subscriptions
));
protected async onMessage(msg: FromResultsViewMsg): Promise<void> {
try {
switch (msg.t) {
case 'resultViewLoaded':
this.onWebViewLoaded();
break;
case 'viewSourceFile': {
await jumpToLocation(msg, this.databaseManager, this.logger);
break;
}
case 'toggleDiagnostics': {
if (msg.visible) {
const databaseItem = this.databaseManager.findDatabaseItem(
Uri.parse(msg.databaseUri)
);
if (databaseItem !== undefined) {
await this.showResultsAsDiagnostics(
msg.origResultsPaths,
msg.metadata,
databaseItem
);
}
} else {
// TODO: Only clear diagnostics on the same database.
this._diagnosticCollection.clear();
}
break;
}
case 'changeSort':
await this.changeRawSortState(msg.resultSetName, msg.sortState);
break;
case 'changeInterpretedSort':
await this.changeInterpretedSortState(msg.sortState);
break;
case 'changePage':
if (msg.selectedTable === ALERTS_TABLE_NAME || msg.selectedTable === GRAPH_TABLE_NAME) {
await this.showPageOfInterpretedResults(msg.pageNumber);
}
else {
await this.showPageOfRawResults(
msg.selectedTable,
msg.pageNumber,
// When we are in an unsorted state, we guarantee that
// sortedResultsInfo doesn't have an entry for the current
// result set. Use this to determine whether or not we use
// the sorted bqrs file.
!!this._displayedQuery?.completedQuery.sortedResultsInfo[msg.selectedTable]
);
}
break;
case 'openFile':
await this.openFile(msg.filePath);
break;
default:
assertNever(msg);
}
} catch (e) {
void showAndLogErrorMessage(getErrorMessage(e), {
fullMessage: getErrorStack(e)
});
}
return this._panel;
}
/**
@@ -296,85 +313,6 @@ export class InterfaceManager extends DisposableObject {
await this.showPageOfRawResults(resultSetName, 0, true);
}
private async handleMsgFromView(msg: FromResultsViewMsg): Promise<void> {
try {
switch (msg.t) {
case 'viewSourceFile': {
await jumpToLocation(msg, this.databaseManager, this.logger);
break;
}
case 'toggleDiagnostics': {
if (msg.visible) {
const databaseItem = this.databaseManager.findDatabaseItem(
Uri.parse(msg.databaseUri)
);
if (databaseItem !== undefined) {
await this.showResultsAsDiagnostics(
msg.origResultsPaths,
msg.metadata,
databaseItem
);
}
} else {
// TODO: Only clear diagnostics on the same database.
this._diagnosticCollection.clear();
}
break;
}
case 'resultViewLoaded':
this._panelLoaded = true;
this._panelLoadedCallBacks.forEach((cb) => cb());
this._panelLoadedCallBacks = [];
break;
case 'changeSort':
await this.changeRawSortState(msg.resultSetName, msg.sortState);
break;
case 'changeInterpretedSort':
await this.changeInterpretedSortState(msg.sortState);
break;
case 'changePage':
if (msg.selectedTable === ALERTS_TABLE_NAME || msg.selectedTable === GRAPH_TABLE_NAME) {
await this.showPageOfInterpretedResults(msg.pageNumber);
}
else {
await this.showPageOfRawResults(
msg.selectedTable,
msg.pageNumber,
// When we are in an unsorted state, we guarantee that
// sortedResultsInfo doesn't have an entry for the current
// result set. Use this to determine whether or not we use
// the sorted bqrs file.
!!this._displayedQuery?.completedQuery.sortedResultsInfo[msg.selectedTable]
);
}
break;
case 'openFile':
await this.openFile(msg.filePath);
break;
default:
assertNever(msg);
}
} catch (e) {
void showAndLogErrorMessage(getErrorMessage(e), {
fullMessage: getErrorStack(e)
});
}
}
postMessage(msg: IntoResultsViewMsg): Thenable<boolean> {
return this.getPanel().webview.postMessage(msg);
}
private waitForPanelLoaded(): Promise<void> {
return new Promise((resolve) => {
if (this._panelLoaded) {
resolve();
} else {
this._panelLoadedCallBacks.push(resolve);
}
});
}
/**
* Show query results in webview panel.
* @param fullQuery Evaluation info for the executed query.

View File

@@ -0,0 +1,460 @@
import * as I from 'immutable';
import { EvaluationLogProblemReporter, EvaluationLogScanner, EvaluationLogScannerProvider } from './log-scanner';
import { InLayer, ComputeRecursive, SummaryEvent, PipelineRun, ComputeSimple } from './log-summary';
const DEFAULT_WARNING_THRESHOLD = 50;
/**
* Like `max`, but returns 0 if no meaningful maximum can be computed.
*/
function safeMax(it: Iterable<number>) {
const m = Math.max(...it);
return Number.isFinite(m) ? m : 0;
}
/**
* Compute a key for the maps that that is sent to report generation.
* Should only be used on events that are known to define queryCausingWork.
*/
function makeKey(
queryCausingWork: string | undefined,
predicate: string,
suffix = ''
): string {
if (queryCausingWork === undefined) {
throw new Error(
'queryCausingWork was not defined on an event we expected it to be defined for!'
);
}
return `${queryCausingWork}:${predicate}${suffix ? ' ' + suffix : ''}`;
}
const DEPENDENT_PREDICATES_REGEXP = (() => {
const regexps = [
// SCAN id
String.raw`SCAN\s+([0-9a-zA-Z:#_]+)\s`,
// JOIN id WITH id
String.raw`JOIN\s+([0-9a-zA-Z:#_]+)\s+WITH\s+([0-9a-zA-Z:#_]+)\s`,
// AGGREGATE id, id
String.raw`AGGREGATE\s+([0-9a-zA-Z:#_]+)\s*,\s+([0-9a-zA-Z:#_]+)`,
// id AND NOT id
String.raw`([0-9a-zA-Z:#_]+)\s+AND\s+NOT\s+([0-9a-zA-Z:#_]+)`,
// INVOKE HIGHER-ORDER RELATION rel ON <id, ..., id>
String.raw`INVOKE\s+HIGHER-ORDER\s+RELATION\s[^\s]+\sON\s+<([0-9a-zA-Z:#_<>]+)((?:,[0-9a-zA-Z:#_<>]+)*)>`,
// SELECT id
String.raw`SELECT\s+([0-9a-zA-Z:#_]+)`
];
return new RegExp(
`${String.raw`\{[0-9]+\}\s+[0-9a-zA-Z]+\s=\s(?:` + regexps.join('|')})`
);
})();
function getDependentPredicates(operations: string[]): I.List<string> {
return I.List(operations).flatMap(operation => {
const matches = DEPENDENT_PREDICATES_REGEXP.exec(operation.trim());
if (matches !== null) {
return I.List(matches)
.rest() // Skip the first group as it's just the entire string
.filter(x => !!x && !x.match('r[0-9]+|PRIMITIVE')) // Only keep the references to predicates.
.flatMap(x => x.split(',')) // Group 2 in the INVOKE HIGHER_ORDER RELATION case is a comma-separated list of identifiers.
.filter(x => !!x); // Remove empty strings
} else {
return I.List();
}
});
}
function getMainHash(event: InLayer | ComputeRecursive): string {
switch (event.evaluationStrategy) {
case 'IN_LAYER':
return event.mainHash;
case 'COMPUTE_RECURSIVE':
return event.raHash;
}
}
/**
* Sum arrays a and b element-wise. The shorter array is padded with 0s if the arrays are not the same length.
*/
function pointwiseSum(a: Int32Array, b: Int32Array, problemReporter: EvaluationLogProblemReporter): Int32Array {
function reportIfInconsistent(ai: number, bi: number) {
if (ai === -1 && bi !== -1) {
problemReporter.log(
`Operation was not evaluated in the first pipeline, but it was evaluated in the accumulated pipeline (with tuple count ${bi}).`
);
}
if (ai !== -1 && bi === -1) {
problemReporter.log(
`Operation was evaluated in the first pipeline (with tuple count ${ai}), but it was not evaluated in the accumulated pipeline.`
);
}
}
const length = Math.max(a.length, b.length);
const result = new Int32Array(length);
for (let i = 0; i < length; i++) {
const ai = a[i] || 0;
const bi = b[i] || 0;
// -1 is used to represent the absence of a tuple count for a line in the pretty-printed RA (e.g. an empty line), so we ignore those.
if (i < a.length && i < b.length && (ai === -1 || bi === -1)) {
result[i] = -1;
reportIfInconsistent(ai, bi);
} else {
result[i] = ai + bi;
}
}
return result;
}
function pushValue<K, V>(m: Map<K, V[]>, k: K, v: V) {
if (!m.has(k)) {
m.set(k, []);
}
m.get(k)!.push(v);
return m;
}
function computeJoinOrderBadness(
maxTupleCount: number,
maxDependentPredicateSize: number,
resultSize: number
): number {
return maxTupleCount / Math.max(maxDependentPredicateSize, resultSize);
}
/**
* A bucket contains the pointwise sum of the tuple counts, result sizes and dependent predicate sizes
* For each (predicate, order) in an SCC, we will compute a bucket.
*/
interface Bucket {
tupleCounts: Int32Array;
resultSize: number;
dependentPredicateSizes: I.Map<string, number>;
}
class JoinOrderScanner implements EvaluationLogScanner {
// Map a predicate hash to its result size
private readonly predicateSizes = new Map<string, number>();
private readonly layerEvents = new Map<string, (ComputeRecursive | InLayer)[]>();
// Map a key of the form 'query-with-demand : predicate name' to its badness input.
private readonly maxTupleCountMap = new Map<string, number[]>();
private readonly resultSizeMap = new Map<string, number[]>();
private readonly maxDependentPredicateSizeMap = new Map<string, number[]>();
private readonly joinOrderMetricMap = new Map<string, number>();
constructor(
private readonly problemReporter: EvaluationLogProblemReporter,
private readonly warningThreshold: number) {
}
public onEvent(event: SummaryEvent): void {
if (
event.completionType !== undefined &&
event.completionType !== 'SUCCESS'
) {
return; // Skip any evaluation that wasn't successful
}
this.recordPredicateSizes(event);
this.computeBadnessMetric(event);
}
public onDone(): void {
void this;
}
private recordPredicateSizes(event: SummaryEvent): void {
switch (event.evaluationStrategy) {
case 'EXTENSIONAL':
case 'COMPUTED_EXTENSIONAL':
case 'COMPUTE_SIMPLE':
case 'CACHACA':
case 'CACHE_HIT': {
this.predicateSizes.set(event.raHash, event.resultSize);
break;
}
case 'SENTINEL_EMPTY': {
this.predicateSizes.set(event.raHash, 0);
break;
}
case 'COMPUTE_RECURSIVE':
case 'IN_LAYER': {
this.predicateSizes.set(event.raHash, event.resultSize);
// layerEvents are indexed by the mainHash.
const hash = getMainHash(event);
if (!this.layerEvents.has(hash)) {
this.layerEvents.set(hash, []);
}
this.layerEvents.get(hash)!.push(event);
break;
}
}
}
private reportProblemIfNecessary(event: SummaryEvent, iteration: number, metric: number): void {
if (metric >= this.warningThreshold) {
this.problemReporter.reportProblem(event.predicateName, event.raHash, iteration,
`Relation '${event.predicateName}' has an inefficient join order. Its join order metric is ${metric.toFixed(2)}, which is larger than the threshold of ${this.warningThreshold.toFixed(2)}.`);
}
}
private computeBadnessMetric(event: SummaryEvent): void {
if (
event.completionType !== undefined &&
event.completionType !== 'SUCCESS'
) {
return; // Skip any evaluation that wasn't successful
}
switch (event.evaluationStrategy) {
case 'COMPUTE_SIMPLE': {
if (!event.pipelineRuns) {
// skip if the optional pipelineRuns field is not present.
break;
}
// Compute the badness metric for a non-recursive predicate. The metric in this case is defined as:
// badness = (max tuple count in the pipeline) / (largest predicate this pipeline depends on)
const key = makeKey(event.queryCausingWork, event.predicateName);
const resultSize = event.resultSize;
// There is only one entry in `pipelineRuns` if it's a non-recursive predicate.
const { maxTupleCount, maxDependentPredicateSize } =
this.badnessInputsForNonRecursiveDelta(event.pipelineRuns[0], event);
if (maxDependentPredicateSize > 0) {
pushValue(this.maxTupleCountMap, key, maxTupleCount);
pushValue(this.resultSizeMap, key, resultSize);
pushValue(
this.maxDependentPredicateSizeMap,
key,
maxDependentPredicateSize
);
const metric = computeJoinOrderBadness(maxTupleCount, maxDependentPredicateSize, resultSize!);
this.joinOrderMetricMap.set(key, metric);
this.reportProblemIfNecessary(event, 0, metric);
}
break;
}
case 'COMPUTE_RECURSIVE': {
// Compute the badness metric for a recursive predicate for each ordering.
const sccMetricInput = this.badnessInputsForRecursiveDelta(event);
// Loop through each predicate in the SCC
sccMetricInput.forEach((buckets, predicate) => {
// Loop through each ordering of the predicate
buckets.forEach((bucket, raReference) => {
// Format the key as demanding-query:name (ordering)
const key = makeKey(
event.queryCausingWork,
predicate,
`(${raReference})`
);
const maxTupleCount = Math.max(...bucket.tupleCounts);
const resultSize = bucket.resultSize;
const maxDependentPredicateSize = Math.max(
...bucket.dependentPredicateSizes.values()
);
if (maxDependentPredicateSize > 0) {
pushValue(this.maxTupleCountMap, key, maxTupleCount);
pushValue(this.resultSizeMap, key, resultSize);
pushValue(
this.maxDependentPredicateSizeMap,
key,
maxDependentPredicateSize
);
const metric = computeJoinOrderBadness(maxTupleCount, maxDependentPredicateSize, resultSize);
const oldMetric = this.joinOrderMetricMap.get(key);
if ((oldMetric === undefined) || (metric > oldMetric)) {
this.joinOrderMetricMap.set(key, metric);
}
}
});
});
break;
}
}
}
/**
* Iterate through an SCC with main node `event`.
*/
private iterateSCC(
event: ComputeRecursive,
func: (
inLayerEvent: ComputeRecursive | InLayer,
run: PipelineRun,
iteration: number
) => void
): void {
const sccEvents = this.layerEvents.get(event.raHash)!;
const nextPipeline: number[] = new Array(sccEvents.length).fill(0);
const maxIteration = Math.max(
...sccEvents.map(e => e.predicateIterationMillis.length)
);
for (let iteration = 0; iteration < maxIteration; ++iteration) {
// Loop through each predicate in this iteration
for (let predicate = 0; predicate < sccEvents.length; ++predicate) {
const inLayerEvent = sccEvents[predicate];
const iterationTime =
inLayerEvent.predicateIterationMillis.length <= iteration
? -1
: inLayerEvent.predicateIterationMillis[iteration];
if (iterationTime != -1) {
const run: PipelineRun =
inLayerEvent.pipelineRuns[nextPipeline[predicate]++];
func(inLayerEvent, run, iteration);
}
}
}
}
/**
* Compute the maximum tuple count and maximum dependent predicate size for a non-recursive pipeline
*/
private badnessInputsForNonRecursiveDelta(
pipelineRun: PipelineRun,
event: ComputeSimple
): { maxTupleCount: number; maxDependentPredicateSize: number } {
const dependentPredicateSizes = Object.values(event.dependencies).map(hash =>
this.predicateSizes.get(hash) ?? 0 // Should always be present, but zero is a safe default.
);
const maxDependentPredicateSize = safeMax(dependentPredicateSizes);
return {
maxTupleCount: safeMax(pipelineRun.counts),
maxDependentPredicateSize: maxDependentPredicateSize
};
}
private prevDeltaSizes(event: ComputeRecursive, predicate: string, i: number) {
// If an iteration isn't present in the map it means it was skipped because the optimizer
// inferred that it was empty. So its size is 0.
return this.curDeltaSizes(event, predicate, i - 1);
}
private curDeltaSizes(event: ComputeRecursive, predicate: string, i: number) {
// If an iteration isn't present in the map it means it was skipped because the optimizer
// inferred that it was empty. So its size is 0.
return (
this.layerEvents.get(event.raHash)?.find(x => x.predicateName === predicate)?.deltaSizes[i] ?? 0
);
}
/**
* Compute the metric dependent predicate sizes and the result size for a predicate in an SCC.
*/
private badnessInputsForLayer(
event: ComputeRecursive,
inLayerEvent: InLayer | ComputeRecursive,
raReference: string,
iteration: number
) {
const dependentPredicates = getDependentPredicates(
inLayerEvent.ra[raReference]
);
let dependentPredicateSizes: I.Map<string, number>;
// We treat the base case as a non-recursive pipeline. In that case, the dependent predicates are
// the dependencies of the base case and the cur_deltas.
if (raReference === 'base') {
dependentPredicateSizes = I.Map(
dependentPredicates.map((pred): [string, number] => {
// A base case cannot contain a `prev_delta`, but it can contain a `cur_delta`.
let size = 0;
if (pred.endsWith('#cur_delta')) {
size = this.curDeltaSizes(
event,
pred.slice(0, -'#cur_delta'.length),
iteration
);
} else {
const hash = event.dependencies[pred];
size = this.predicateSizes.get(hash)!;
}
return [pred, size];
})
);
} else {
// It's a non-base case in a recursive pipeline. In that case, the dependent predicates are
// only the prev_deltas.
dependentPredicateSizes = I.Map(
dependentPredicates
.flatMap(pred => {
// If it's actually a prev_delta
if (pred.endsWith('#prev_delta')) {
// Return the predicate without the #prev_delta suffix.
return [pred.slice(0, -'#prev_delta'.length)];
} else {
// Not a recursive delta. Skip it.
return [];
}
})
.map((prev): [string, number] => {
const size = this.prevDeltaSizes(event, prev, iteration);
return [prev, size];
})
);
}
const deltaSize = inLayerEvent.deltaSizes[iteration];
return { dependentPredicateSizes, deltaSize };
}
/**
* Compute the metric input for all the events in a SCC that starts with main node `event`
*/
private badnessInputsForRecursiveDelta(event: ComputeRecursive): Map<string, Map<string, Bucket>> {
// nameToOrderToBucket : predicate name -> ordering (i.e., standard, order_500000, etc.) -> bucket
const nameToOrderToBucket = new Map<string, Map<string, Bucket>>();
// Iterate through the SCC and compute the metric inputs
this.iterateSCC(event, (inLayerEvent, run, iteration) => {
const raReference = run.raReference;
const predicateName = inLayerEvent.predicateName;
if (!nameToOrderToBucket.has(predicateName)) {
nameToOrderToBucket.set(predicateName, new Map());
}
const orderTobucket = nameToOrderToBucket.get(predicateName)!;
if (!orderTobucket.has(raReference)) {
orderTobucket.set(raReference, {
tupleCounts: new Int32Array(0),
resultSize: 0,
dependentPredicateSizes: I.Map()
});
}
const { dependentPredicateSizes, deltaSize } = this.badnessInputsForLayer(
event,
inLayerEvent,
raReference,
iteration
);
const bucket = orderTobucket.get(raReference)!;
// Pointwise sum the tuple counts
const newTupleCounts = pointwiseSum(
bucket.tupleCounts,
new Int32Array(run.counts),
this.problemReporter
);
const resultSize = bucket.resultSize + deltaSize;
// Pointwise sum the deltas.
const newDependentPredicateSizes = bucket.dependentPredicateSizes.mergeWith(
(oldSize, newSize) => oldSize + newSize,
dependentPredicateSizes
);
orderTobucket.set(raReference, {
tupleCounts: newTupleCounts,
resultSize: resultSize,
dependentPredicateSizes: newDependentPredicateSizes
});
});
return nameToOrderToBucket;
}
}
export class JoinOrderScannerProvider implements EvaluationLogScannerProvider {
public createScanner(problemReporter: EvaluationLogProblemReporter): EvaluationLogScanner {
return new JoinOrderScanner(problemReporter, DEFAULT_WARNING_THRESHOLD);
}
}

View File

@@ -0,0 +1,23 @@
import * as fs from 'fs-extra';
/**
* Read a file consisting of multiple JSON objects. Each object is separated from the previous one
* by a double newline sequence. This is basically a more human-readable form of JSONL.
*
* The current implementation reads the entire text of the document into memory, but in the future
* it will stream the document to improve the performance with large documents.
*
* @param path The path to the file.
* @param handler Callback to be invoked for each top-level JSON object in order.
*/
export async function readJsonlFile(path: string, handler: (value: any) => Promise<void>): Promise<void> {
const logSummary = await fs.readFile(path, 'utf-8');
// Remove newline delimiters because summary is in .jsonl format.
const jsonSummaryObjects: string[] = logSummary.split(/\r?\n\r?\n/g);
for (const obj of jsonSummaryObjects) {
const jsonObj = JSON.parse(obj);
await handler(jsonObj);
}
}

View File

@@ -0,0 +1,109 @@
import { Diagnostic, DiagnosticSeverity, languages, Range, Uri } from 'vscode';
import { DisposableObject } from '../pure/disposable-object';
import { QueryHistoryManager } from '../query-history';
import { QueryHistoryInfo } from '../query-results';
import { EvaluationLogProblemReporter, EvaluationLogScannerSet } from './log-scanner';
import { PipelineInfo, SummarySymbols } from './summary-parser';
import * as fs from 'fs-extra';
import { logger } from '../logging';
/**
* Compute the key used to find a predicate in the summary symbols.
* @param name The name of the predicate.
* @param raHash The RA hash of the predicate.
* @returns The key of the predicate, consisting of `name@shortHash`, where `shortHash` is the first
* eight characters of `raHash`.
*/
function predicateSymbolKey(name: string, raHash: string): string {
return `${name}@${raHash.substring(0, 8)}`;
}
/**
* Implementation of `EvaluationLogProblemReporter` that generates `Diagnostic` objects to display
* in the VS Code "Problems" view.
*/
class ProblemReporter implements EvaluationLogProblemReporter {
public readonly diagnostics: Diagnostic[] = [];
constructor(private readonly symbols: SummarySymbols | undefined) {
}
public reportProblem(predicateName: string, raHash: string, iteration: number, message: string): void {
const nameWithHash = predicateSymbolKey(predicateName, raHash);
const predicateSymbol = this.symbols?.predicates[nameWithHash];
let predicateInfo: PipelineInfo | undefined = undefined;
if (predicateSymbol !== undefined) {
predicateInfo = predicateSymbol.iterations[iteration];
}
if (predicateInfo !== undefined) {
const range = new Range(predicateInfo.raStartLine, 0, predicateInfo.raEndLine + 1, 0);
this.diagnostics.push(new Diagnostic(range, message, DiagnosticSeverity.Error));
}
}
public log(message: string): void {
void logger.log(message);
}
}
export class LogScannerService extends DisposableObject {
public readonly scanners = new EvaluationLogScannerSet();
private readonly diagnosticCollection = this.push(languages.createDiagnosticCollection('ql-eval-log'));
private currentItem: QueryHistoryInfo | undefined = undefined;
constructor(qhm: QueryHistoryManager) {
super();
this.push(qhm.onDidChangeCurrentQueryItem(async (item) => {
if (item !== this.currentItem) {
this.currentItem = item;
await this.scanEvalLog(item);
}
}));
this.push(qhm.onDidCompleteQuery(async (item) => {
if (item === this.currentItem) {
await this.scanEvalLog(item);
}
}));
}
/**
* Scan the evaluation log for a query, and report any diagnostics.
*
* @param query The query whose log is to be scanned.
*/
public async scanEvalLog(
query: QueryHistoryInfo | undefined
): Promise<void> {
this.diagnosticCollection.clear();
if ((query?.t !== 'local')
|| (query.evalLogSummaryLocation === undefined)
|| (query.jsonEvalLogSummaryLocation === undefined)) {
return;
}
const diagnostics = await this.scanLog(query.jsonEvalLogSummaryLocation, query.evalLogSummarySymbolsLocation);
const uri = Uri.file(query.evalLogSummaryLocation);
this.diagnosticCollection.set(uri, diagnostics);
}
/**
* Scan the evaluator summary log for problems, using the scanners for all registered providers.
* @param jsonSummaryLocation The file path of the JSON summary log.
* @param symbolsLocation The file path of the symbols file for the human-readable log summary.
* @returns An array of `Diagnostic`s representing the problems found by scanners.
*/
private async scanLog(jsonSummaryLocation: string, symbolsLocation: string | undefined): Promise<Diagnostic[]> {
let symbols: SummarySymbols | undefined = undefined;
if (symbolsLocation !== undefined) {
symbols = JSON.parse(await fs.readFile(symbolsLocation, { encoding: 'utf-8' }));
}
const problemReporter = new ProblemReporter(symbols);
await this.scanners.scanLog(jsonSummaryLocation, problemReporter);
return problemReporter.diagnostics;
}
}

View File

@@ -0,0 +1,103 @@
import { SummaryEvent } from './log-summary';
import { readJsonlFile } from './jsonl-reader';
/**
* Callback interface used to report diagnostics from a log scanner.
*/
export interface EvaluationLogProblemReporter {
/**
* Report a potential problem detected in the evaluation log.
*
* @param predicateName The mangled name of the predicate with the problem.
* @param raHash The RA hash of the predicate with the problem.
* @param iteration The iteration number with the problem. For a non-recursive predicate, this
* must be zero.
* @param message The problem message.
*/
reportProblem(predicateName: string, raHash: string, iteration: number, message: string): void;
/**
* Log a message about a problem in the implementation of the scanner. These will typically be
* displayed separate from any problems reported via `reportProblem()`.
*/
log(message: string): void;
}
/**
* Interface implemented by a log scanner. Instances are created via
* `EvaluationLogScannerProvider.createScanner()`.
*/
export interface EvaluationLogScanner {
/**
* Called for each event in the log summary, in order. The implementation can report problems via
* the `EvaluationLogProblemReporter` interface that was supplied to `createScanner()`.
* @param event The log summary event.
*/
onEvent(event: SummaryEvent): void;
/**
* Called after all events in the log summary have been processed. The implementation can report
* problems via the `EvaluationLogProblemReporter` interface that was supplied to
* `createScanner()`.
*/
onDone(): void;
}
/**
* A factory for log scanners. When a log is to be scanned, all registered
* `EvaluationLogScannerProviders` will be asked to create a new instance of `EvaluationLogScanner`
* to do the scanning.
*/
export interface EvaluationLogScannerProvider {
/**
* Create a new instance of `EvaluationLogScanner` to scan a single summary log.
* @param problemReporter Callback interface for reporting any problems discovered.
*/
createScanner(problemReporter: EvaluationLogProblemReporter): EvaluationLogScanner;
}
/**
* Same as VSCode's `Disposable`, but avoids a dependency on VS Code.
*/
export interface Disposable {
dispose(): void;
}
export class EvaluationLogScannerSet {
private readonly scannerProviders = new Map<number, EvaluationLogScannerProvider>();
private nextScannerProviderId = 0;
/**
* Register a provider that can create instances of `EvaluationLogScanner` to scan evaluation logs
* for problems.
* @param provider The provider.
* @returns A `Disposable` that, when disposed, will unregister the provider.
*/
public registerLogScannerProvider(provider: EvaluationLogScannerProvider): Disposable {
const id = this.nextScannerProviderId;
this.nextScannerProviderId++;
this.scannerProviders.set(id, provider);
return {
dispose: () => {
this.scannerProviders.delete(id);
}
};
}
/**
* Scan the evaluator summary log for problems, using the scanners for all registered providers.
* @param jsonSummaryLocation The file path of the JSON summary log.
* @param problemReporter Callback interface for reporting any problems discovered.
*/
public async scanLog(jsonSummaryLocation: string, problemReporter: EvaluationLogProblemReporter): Promise<void> {
const scanners = [...this.scannerProviders.values()].map(p => p.createScanner(problemReporter));
await readJsonlFile(jsonSummaryLocation, async obj => {
scanners.forEach(scanner => {
scanner.onEvent(obj);
});
});
scanners.forEach(scanner => scanner.onDone());
}
}

View File

@@ -0,0 +1,93 @@
export interface PipelineRun {
raReference: string;
counts: number[];
duplicationPercentages: number[];
}
export interface Ra {
[key: string]: string[];
}
export type EvaluationStrategy =
'COMPUTE_SIMPLE' |
'COMPUTE_RECURSIVE' |
'IN_LAYER' |
'COMPUTED_EXTENSIONAL' |
'EXTENSIONAL' |
'SENTINEL_EMPTY' |
'CACHACA' |
'CACHE_HIT';
interface SummaryEventBase {
evaluationStrategy: EvaluationStrategy;
predicateName: string;
raHash: string;
appearsAs: { [key: string]: { [key: string]: number[] } };
completionType?: string;
}
interface ResultEventBase extends SummaryEventBase {
resultSize: number;
}
export interface ComputeSimple extends ResultEventBase {
evaluationStrategy: 'COMPUTE_SIMPLE';
ra: Ra;
pipelineRuns?: [PipelineRun];
queryCausingWork?: string;
dependencies: { [key: string]: string };
}
export interface ComputeRecursive extends ResultEventBase {
evaluationStrategy: 'COMPUTE_RECURSIVE';
deltaSizes: number[];
ra: Ra;
pipelineRuns: PipelineRun[];
queryCausingWork?: string;
dependencies: { [key: string]: string };
predicateIterationMillis: number[];
}
export interface InLayer extends ResultEventBase {
evaluationStrategy: 'IN_LAYER';
deltaSizes: number[];
ra: Ra;
pipelineRuns: PipelineRun[];
queryCausingWork?: string;
mainHash: string;
predicateIterationMillis: number[];
}
export interface ComputedExtensional extends ResultEventBase {
evaluationStrategy: 'COMPUTED_EXTENSIONAL';
queryCausingWork?: string;
}
export interface NonComputedExtensional extends ResultEventBase {
evaluationStrategy: 'EXTENSIONAL';
queryCausingWork?: string;
}
export interface SentinelEmpty extends SummaryEventBase {
evaluationStrategy: 'SENTINEL_EMPTY';
sentinelRaHash: string;
}
export interface Cachaca extends ResultEventBase {
evaluationStrategy: 'CACHACA';
}
export interface CacheHit extends ResultEventBase {
evaluationStrategy: 'CACHE_HIT';
}
export type Extensional = ComputedExtensional | NonComputedExtensional;
export type SummaryEvent =
| ComputeSimple
| ComputeRecursive
| InLayer
| Extensional
| SentinelEmpty
| Cachaca
| CacheHit;

View File

@@ -0,0 +1,154 @@
import * as fs from 'fs-extra';
import { RawSourceMap, SourceMapConsumer } from 'source-map';
import { commands, Position, Selection, TextDocument, TextEditor, TextEditorRevealType, TextEditorSelectionChangeEvent, ViewColumn, window, workspace } from 'vscode';
import { DisposableObject } from '../pure/disposable-object';
import { commandRunner } from '../commandRunner';
import { logger } from '../logging';
import { getErrorMessage } from '../pure/helpers-pure';
/** A `Position` within a specified file on disk. */
interface PositionInFile {
filePath: string;
position: Position;
}
/**
* Opens the specified source location in a text editor.
* @param position The position (including file path) to show.
*/
async function showSourceLocation(position: PositionInFile): Promise<void> {
const document = await workspace.openTextDocument(position.filePath);
const editor = await window.showTextDocument(document, ViewColumn.Active);
editor.selection = new Selection(position.position, position.position);
editor.revealRange(editor.selection, TextEditorRevealType.InCenterIfOutsideViewport);
}
/**
* Simple language support for human-readable evaluator log summaries.
*
* This class implements the `codeQL.gotoQL` command, which jumps from RA code to the corresponding
* QL code that generated it. It also tracks the current selection and active editor to enable and
* disable that command based on whether there is a QL mapping for the current selection.
*/
export class SummaryLanguageSupport extends DisposableObject {
/**
* The last `TextDocument` (with language `ql-summary`) for which we tried to find a sourcemap, or
* `undefined` if we have not seen such a document yet.
*/
private lastDocument: TextDocument | undefined = undefined;
/**
* The sourcemap for `lastDocument`, or `undefined` if there was no such sourcemap or document.
*/
private sourceMap: SourceMapConsumer | undefined = undefined;
constructor() {
super();
this.push(window.onDidChangeActiveTextEditor(this.handleDidChangeActiveTextEditor));
this.push(window.onDidChangeTextEditorSelection(this.handleDidChangeTextEditorSelection));
this.push(workspace.onDidCloseTextDocument(this.handleDidCloseTextDocument));
this.push(commandRunner('codeQL.gotoQL', this.handleGotoQL));
}
/**
* Gets the location of the QL code that generated the RA at the current selection in the active
* editor, or `undefined` if there is no mapping.
*/
private async getQLSourceLocation(): Promise<PositionInFile | undefined> {
const editor = window.activeTextEditor;
if (editor === undefined) {
return undefined;
}
const document = editor.document;
if (document.languageId !== 'ql-summary') {
return undefined;
}
if (document.uri.scheme !== 'file') {
return undefined;
}
if (this.lastDocument !== document) {
this.clearCache();
const mapPath = document.uri.fsPath + '.map';
try {
const sourceMapText = await fs.readFile(mapPath, 'utf-8');
const rawMap: RawSourceMap = JSON.parse(sourceMapText);
this.sourceMap = await new SourceMapConsumer(rawMap);
} catch (e: unknown) {
// Error reading sourcemap. Pretend there was no sourcemap.
void logger.log(`Error reading sourcemap file '${mapPath}': ${getErrorMessage(e)}`);
this.sourceMap = undefined;
}
this.lastDocument = document;
}
if (this.sourceMap === undefined) {
return undefined;
}
const qlPosition = this.sourceMap.originalPositionFor({
line: editor.selection.start.line + 1,
column: editor.selection.start.character,
bias: SourceMapConsumer.GREATEST_LOWER_BOUND
});
if ((qlPosition.source === null) || (qlPosition.line === null)) {
// No position found.
return undefined;
}
const line = qlPosition.line - 1; // In `source-map`, lines are 1-based...
const column = qlPosition.column ?? 0; // ...but columns are 0-based :(
return {
filePath: qlPosition.source,
position: new Position(line, column)
};
}
/**
* Clears the cached sourcemap and its corresponding `TextDocument`.
*/
private clearCache(): void {
if (this.sourceMap !== undefined) {
this.sourceMap.destroy();
this.sourceMap = undefined;
this.lastDocument = undefined;
}
}
/**
* Updates the `codeql.hasQLSource` context variable based on the current selection. This variable
* controls whether or not the `codeQL.gotoQL` command is enabled.
*/
private async updateContext(): Promise<void> {
const position = await this.getQLSourceLocation();
await commands.executeCommand('setContext', 'codeql.hasQLSource', position !== undefined);
}
handleDidChangeActiveTextEditor = async (_editor: TextEditor | undefined): Promise<void> => {
await this.updateContext();
}
handleDidChangeTextEditorSelection = async (_e: TextEditorSelectionChangeEvent): Promise<void> => {
await this.updateContext();
}
handleDidCloseTextDocument = (document: TextDocument): void => {
if (this.lastDocument === document) {
this.clearCache();
}
}
handleGotoQL = async (): Promise<void> => {
const position = await this.getQLSourceLocation();
if (position !== undefined) {
await showSourceLocation(position);
}
};
}

View File

@@ -0,0 +1,113 @@
import * as fs from 'fs-extra';
/**
* Location information for a single pipeline invocation in the RA.
*/
export interface PipelineInfo {
startLine: number;
raStartLine: number;
raEndLine: number;
}
/**
* Location information for a single predicate in the RA.
*/
export interface PredicateSymbol {
/**
* `PipelineInfo` for each iteration. A non-recursive predicate will have a single iteration `0`.
*/
iterations: Record<number, PipelineInfo>;
}
/**
* Location information for the RA from an evaluation log. Line numbers point into the
* human-readable log summary.
*/
export interface SummarySymbols {
predicates: Record<string, PredicateSymbol>;
}
// Tuple counts for Expr::Expr::getParent#dispred#f0820431#ff@76d6745o:
const NON_RECURSIVE_TUPLE_COUNT_REGEXP = /^Evaluated relational algebra for predicate (?<predicateName>\S+) with tuple counts:$/;
// Tuple counts for Expr::Expr::getEnclosingStmt#f0820431#bf@923ddwj9 on iteration 0 running pipeline base:
const RECURSIVE_TUPLE_COUNT_REGEXP = /^Evaluated relational algebra for predicate (?<predicateName>\S+) on iteration (?<iteration>\d+) running pipeline (?<pipeline>\S+) with tuple counts:$/;
const RETURN_REGEXP = /^\s*return /;
/**
* Parse a human-readable evaluation log summary to find the location of the RA for each pipeline
* run.
*
* TODO: Once we're more certain about the symbol format, we should have the CLI generate this as it
* generates the human-readabe summary to avoid having to rely on regular expression matching of the
* human-readable text.
*
* @param summaryPath The path to the summary file.
* @param symbolsPath The path to the symbols file to generate.
*/
export async function generateSummarySymbolsFile(summaryPath: string, symbolsPath: string): Promise<void> {
const symbols = await generateSummarySymbols(summaryPath);
await fs.writeFile(symbolsPath, JSON.stringify(symbols));
}
/**
* Parse a human-readable evaluation log summary to find the location of the RA for each pipeline
* run.
*
* @param fileLocation The path to the summary file.
* @returns Symbol information for the summary file.
*/
async function generateSummarySymbols(summaryPath: string): Promise<SummarySymbols> {
const summary = await fs.promises.readFile(summaryPath, { encoding: 'utf-8' });
const symbols: SummarySymbols = {
predicates: {}
};
const lines = summary.split(/\r?\n/);
let lineNumber = 0;
while (lineNumber < lines.length) {
const startLineNumber = lineNumber;
lineNumber++;
const startLine = lines[startLineNumber];
const nonRecursiveMatch = startLine.match(NON_RECURSIVE_TUPLE_COUNT_REGEXP);
let predicateName: string | undefined = undefined;
let iteration = 0;
if (nonRecursiveMatch) {
predicateName = nonRecursiveMatch.groups!.predicateName;
} else {
const recursiveMatch = startLine.match(RECURSIVE_TUPLE_COUNT_REGEXP);
if (recursiveMatch?.groups) {
predicateName = recursiveMatch.groups.predicateName;
iteration = parseInt(recursiveMatch.groups.iteration);
}
}
if (predicateName !== undefined) {
const raStartLine = lineNumber;
let raEndLine: number | undefined = undefined;
while ((lineNumber < lines.length) && (raEndLine === undefined)) {
const raLine = lines[lineNumber];
const returnMatch = raLine.match(RETURN_REGEXP);
if (returnMatch) {
raEndLine = lineNumber;
}
lineNumber++;
}
if (raEndLine !== undefined) {
let symbol = symbols.predicates[predicateName];
if (symbol === undefined) {
symbol = {
iterations: {}
};
symbols.predicates[predicateName] = symbol;
}
symbol.iterations[iteration] = {
startLine: lineNumber,
raStartLine: raStartLine,
raEndLine: raEndLine
};
}
}
}
return symbols;
}

View File

@@ -103,7 +103,14 @@ export function transformBqrsResultSet(
};
}
type BqrsKind = 'String' | 'Float' | 'Integer' | 'String' | 'Boolean' | 'Date' | 'Entity';
interface BqrsColumn {
name: string;
kind: BqrsKind;
}
export interface DecodedBqrsChunk {
tuples: CellValue[][];
next?: number;
columns: BqrsColumn[];
}

View File

@@ -97,20 +97,35 @@ export function isStringLoc(loc: UrlValue): loc is string {
export function tryGetRemoteLocation(
loc: UrlValue | undefined,
fileLinkPrefix: string
fileLinkPrefix: string,
sourceLocationPrefix: string | undefined,
): string | undefined {
const resolvableLocation = tryGetResolvableLocation(loc);
if (!resolvableLocation) {
return undefined;
}
// Remote locations have the following format:
// file:/home/runner/work/<repo>/<repo/relative/path/to/file
// So we need to drop the first 6 parts of the path.
let trimmedLocation: string;
// TODO: We can make this more robust to other path formats.
const locationParts = resolvableLocation.uri.split('/');
const trimmedLocation = locationParts.slice(6, locationParts.length).join('/');
// Remote locations have the following format:
// "file:${sourceLocationPrefix}/relative/path/to/file"
// So we need to strip off the first part to get the relative path.
if (sourceLocationPrefix) {
if (!resolvableLocation.uri.startsWith(`file:${sourceLocationPrefix}/`)) {
return undefined;
}
trimmedLocation = resolvableLocation.uri.replace(`file:${sourceLocationPrefix}/`, '');
} else {
// If the source location prefix is empty (e.g. for older remote queries), we assume that the database
// was created on a Linux actions runner and has the format:
// "file:/home/runner/work/<repo>/<repo>/relative/path/to/file"
// So we need to drop the first 6 parts of the path.
if (!resolvableLocation.uri.startsWith('file:/home/runner/work/')) {
return undefined;
}
const locationParts = resolvableLocation.uri.split('/');
trimmedLocation = locationParts.slice(6, locationParts.length).join('/');
}
const fileLink = {
fileLinkPrefix,

View File

@@ -31,17 +31,18 @@ export const asyncFilter = async function <T>(arr: T[], predicate: (arg0: T) =>
return arr.filter((_, index) => results[index]);
};
export const ONE_DAY_IN_MS = 24 * 60 * 60 * 1000;
export const ONE_HOUR_IN_MS = 1000 * 60 * 60;
export const TWO_HOURS_IN_MS = 1000 * 60 * 60 * 2;
export const THREE_HOURS_IN_MS = 1000 * 60 * 60 * 3;
/**
* This regex matches strings of the form `owner/repo` where:
* - `owner` is made up of alphanumeric characters or single hyphens, starting and ending in an alphanumeric character
* - `repo` is made up of alphanumeric characters, hyphens, or underscores
* - `owner` is made up of alphanumeric characters, hyphens, underscores, or periods
* - `repo` is made up of alphanumeric characters, hyphens, underscores, or periods
*/
export const REPO_REGEX = /^(?:[a-zA-Z0-9]+-)*[a-zA-Z0-9]+\/[a-zA-Z0-9-_]+$/;
export const REPO_REGEX = /^[a-zA-Z0-9-_\.]+\/[a-zA-Z0-9-_\.]+$/;
/**
* This regex matches GiHub organization and user strings. These are made up for alphanumeric
* characters, hyphens, underscores or periods.
*/
export const OWNER_REGEX = /^[a-zA-Z0-9-_\.]+$/;
export function getErrorMessage(e: any) {
return e instanceof Error ? e.message : String(e);

View File

@@ -395,7 +395,8 @@ export type FromRemoteQueriesMessage =
| OpenVirtualFileMsg
| RemoteQueryDownloadAnalysisResultsMessage
| RemoteQueryDownloadAllAnalysesResultsMessage
| RemoteQueryExportResultsMessage;
| RemoteQueryExportResultsMessage
| CopyRepoListMessage;
export type ToRemoteQueriesMessage =
| SetRemoteQueryResultMessage
@@ -433,3 +434,8 @@ export interface RemoteQueryDownloadAllAnalysesResultsMessage {
export interface RemoteQueryExportResultsMessage {
t: 'remoteQueryExportResults';
}
export interface CopyRepoListMessage {
t: 'copyRepoList';
queryId: string;
}

View File

@@ -0,0 +1,34 @@
import { readJsonlFile } from '../log-insights/jsonl-reader';
// TODO(angelapwen): Only load in necessary information and
// location in bytes for this log to save memory.
export interface EvalLogData {
predicateName: string;
millis: number;
resultSize: number;
// Key: pipeline identifier; Value: array of pipeline steps
ra: Record<string, string[]>;
}
/**
* A pure method that parses a string of evaluator log summaries into
* an array of EvalLogData objects.
*/
export async function parseViewerData(jsonSummaryPath: string): Promise<EvalLogData[]> {
const viewerData: EvalLogData[] = [];
await readJsonlFile(jsonSummaryPath, async jsonObj => {
// Only convert log items that have an RA and millis field
if (jsonObj.ra !== undefined && jsonObj.millis !== undefined) {
const newLogData: EvalLogData = {
predicateName: jsonObj.predicateName,
millis: jsonObj.millis,
resultSize: jsonObj.resultSize,
ra: jsonObj.ra
};
viewerData.push(newLogData);
}
});
return viewerData;
}

View File

@@ -155,6 +155,10 @@ export interface CompilationOptions {
* get reported anyway. Useful for universal compilation options.
*/
computeDefaultStrings: boolean;
/**
* Emit debug information in compiled query.
*/
emitDebugInfo: boolean;
}
/**
@@ -650,7 +654,7 @@ export interface ClearCacheParams {
/**
* Parameters to start a new structured log
*/
export interface StartLogParams {
export interface StartLogParams {
/**
* The dataset for which we want to start a new structured log
*/
@@ -664,7 +668,7 @@ export interface ClearCacheParams {
/**
* Parameters to terminate a structured log
*/
export interface EndLogParams {
export interface EndLogParams {
/**
* The dataset for which we want to terminated the log
*/
@@ -1070,12 +1074,12 @@ export const compileUpgradeSequence = new rpc.RequestType<WithProgressId<Compile
/**
* Start a new structured log in the evaluator, terminating the previous one if it exists
*/
export const startLog = new rpc.RequestType<WithProgressId<StartLogParams>, StartLogResult, void, void>('evaluation/startLog');
export const startLog = new rpc.RequestType<WithProgressId<StartLogParams>, StartLogResult, void, void>('evaluation/startLog');
/**
* Terminate a structured log in the evaluator. Is a no-op if we aren't logging to the given location
*/
export const endLog = new rpc.RequestType<WithProgressId<EndLogParams>, EndLogResult, void, void>('evaluation/endLog');
export const endLog = new rpc.RequestType<WithProgressId<EndLogParams>, EndLogResult, void, void>('evaluation/endLog');
/**
* Clear the cache of a dataset

View File

@@ -0,0 +1,85 @@
/*
* Contains an assortment of helper constants and functions for working with time, dates, and durations.
*/
export const ONE_MINUTE_IN_MS = 1000 * 60;
export const ONE_HOUR_IN_MS = ONE_MINUTE_IN_MS * 60;
export const TWO_HOURS_IN_MS = ONE_HOUR_IN_MS * 2;
export const THREE_HOURS_IN_MS = ONE_HOUR_IN_MS * 3;
export const ONE_DAY_IN_MS = ONE_HOUR_IN_MS * 24;
// These are approximations
export const ONE_MONTH_IN_MS = ONE_DAY_IN_MS * 30;
export const ONE_YEAR_IN_MS = ONE_DAY_IN_MS * 365;
const durationFormatter = new Intl.RelativeTimeFormat('en', {
numeric: 'auto',
});
/**
* Converts a number of milliseconds into a human-readable string with units, indicating a relative time in the past or future.
*
* @param relativeTimeMillis The duration in milliseconds. A negative number indicates a duration in the past. And a positive number is
* the future.
* @returns A humanized duration. For example, "in 2 minutes", "2 minutes ago", "yesterday", or "tomorrow".
*/
export function humanizeRelativeTime(relativeTimeMillis?: number) {
if (relativeTimeMillis === undefined) {
return '';
}
if (Math.abs(relativeTimeMillis) < ONE_HOUR_IN_MS) {
return durationFormatter.format(Math.floor(relativeTimeMillis / ONE_MINUTE_IN_MS), 'minute');
} else if (Math.abs(relativeTimeMillis) < ONE_DAY_IN_MS) {
return durationFormatter.format(Math.floor(relativeTimeMillis / ONE_HOUR_IN_MS), 'hour');
} else if (Math.abs(relativeTimeMillis) < ONE_MONTH_IN_MS) {
return durationFormatter.format(Math.floor(relativeTimeMillis / ONE_DAY_IN_MS), 'day');
} else if (Math.abs(relativeTimeMillis) < ONE_YEAR_IN_MS) {
return durationFormatter.format(Math.floor(relativeTimeMillis / ONE_MONTH_IN_MS), 'month');
} else {
return durationFormatter.format(Math.floor(relativeTimeMillis / ONE_YEAR_IN_MS), 'year');
}
}
/**
* Converts a number of milliseconds into a human-readable string with units, indicating an amount of time.
* Negative numbers have no meaning and are considered to be "Less than a minute".
*
* @param millis The number of milliseconds to convert.
* @returns A humanized duration. For example, "2 minutes", "2 hours", "2 days", or "2 months".
*/
export function humanizeUnit(millis?: number): string {
// assume a blank or empty string is a zero
// assume anything less than 0 is a zero
if (!millis || millis < ONE_MINUTE_IN_MS) {
return 'Less than a minute';
}
let unit: string;
let unitDiff: number;
if (millis < ONE_HOUR_IN_MS) {
unit = 'minute';
unitDiff = Math.floor(millis / ONE_MINUTE_IN_MS);
} else if (millis < ONE_DAY_IN_MS) {
unit = 'hour';
unitDiff = Math.floor(millis / ONE_HOUR_IN_MS);
} else if (millis < ONE_MONTH_IN_MS) {
unit = 'day';
unitDiff = Math.floor(millis / ONE_DAY_IN_MS);
} else if (millis < ONE_YEAR_IN_MS) {
unit = 'month';
unitDiff = Math.floor(millis / ONE_MONTH_IN_MS);
} else {
unit = 'year';
unitDiff = Math.floor(millis / ONE_YEAR_IN_MS);
}
return createFormatter(unit).format(unitDiff);
}
function createFormatter(unit: string) {
return Intl.NumberFormat('en-US', {
style: 'unit',
unit,
unitDisplay: 'long'
});
}

View File

@@ -3,6 +3,7 @@ import * as os from 'os';
import * as path from 'path';
import { Disposable, ExtensionContext } from 'vscode';
import { logger } from './logging';
import { QueryHistoryManager } from './query-history';
const LAST_SCRUB_TIME_KEY = 'lastScrubTime';
@@ -30,12 +31,13 @@ export function registerQueryHistoryScubber(
throttleTime: number,
maxQueryTime: number,
queryDirectory: string,
qhm: QueryHistoryManager,
ctx: ExtensionContext,
// optional counter to keep track of how many times the scrubber has run
counter?: Counter
): Disposable {
const deregister = setInterval(scrubQueries, wakeInterval, throttleTime, maxQueryTime, queryDirectory, ctx, counter);
const deregister = setInterval(scrubQueries, wakeInterval, throttleTime, maxQueryTime, queryDirectory, qhm, ctx, counter);
return {
dispose: () => {
@@ -48,6 +50,7 @@ async function scrubQueries(
throttleTime: number,
maxQueryTime: number,
queryDirectory: string,
qhm: QueryHistoryManager,
ctx: ExtensionContext,
counter?: Counter
) {
@@ -89,6 +92,7 @@ async function scrubQueries(
} finally {
void logger.log(`Scrubbed ${scrubCount} old queries.`);
}
await qhm.removeDeletedQueries();
}
}

View File

@@ -9,6 +9,7 @@ import {
ProviderResult,
Range,
ThemeIcon,
TreeDataProvider,
TreeItem,
TreeView,
Uri,
@@ -28,7 +29,8 @@ import { URLSearchParams } from 'url';
import { QueryServerClient } from './queryserver-client';
import { DisposableObject } from './pure/disposable-object';
import { commandRunner } from './commandRunner';
import { assertNever, ONE_HOUR_IN_MS, TWO_HOURS_IN_MS, getErrorMessage, getErrorStack } from './pure/helpers-pure';
import { ONE_HOUR_IN_MS, TWO_HOURS_IN_MS } from './pure/time';
import { assertNever, getErrorMessage, getErrorStack } from './pure/helpers-pure';
import { CompletedLocalQueryInfo, LocalQueryInfo as LocalQueryInfo, QueryHistoryInfo } from './query-results';
import { DatabaseManager } from './databases';
import { registerQueryHistoryScubber } from './query-history-scrubber';
@@ -39,6 +41,14 @@ import { CliVersionConstraint } from './cli';
import { HistoryItemLabelProvider } from './history-item-label-provider';
import { Credentials } from './authentication';
import { cancelRemoteQuery } from './remote-queries/gh-actions-api-client';
import { RemoteQueriesManager } from './remote-queries/remote-queries-manager';
import { RemoteQueryHistoryItem } from './remote-queries/remote-query-history-item';
import { InterfaceManager } from './interface';
import { WebviewReveal } from './interface-utils';
import { EvalLogViewer } from './eval-log-viewer';
import EvalLogTreeBuilder from './eval-log-tree-builder';
import { EvalLogData, parseViewerData } from './pure/log-summary-parser';
import { QueryWithResults } from './run-queries';
/**
* query-history.ts
@@ -106,7 +116,7 @@ const WORKSPACE_QUERY_HISTORY_FILE = 'workspace-query-history.json';
/**
* Tree data provider for the query history view.
*/
export class HistoryTreeDataProvider extends DisposableObject {
export class HistoryTreeDataProvider extends DisposableObject implements TreeDataProvider<QueryHistoryInfo> {
private _sortOrder = SortOrder.DateAsc;
private _onDidChangeTreeData = super.push(new EventEmitter<QueryHistoryInfo | undefined>());
@@ -114,6 +124,10 @@ export class HistoryTreeDataProvider extends DisposableObject {
readonly onDidChangeTreeData: Event<QueryHistoryInfo | undefined> = this
._onDidChangeTreeData.event;
private _onDidChangeCurrentQueryItem = super.push(new EventEmitter<QueryHistoryInfo | undefined>());
public readonly onDidChangeCurrentQueryItem = this._onDidChangeCurrentQueryItem.event;
private history: QueryHistoryInfo[] = [];
private failedIconPath: string;
@@ -200,13 +214,12 @@ export class HistoryTreeDataProvider extends DisposableObject {
? h2.initialInfo.start.getTime()
: h2.remoteQuery?.executionStartTime;
// result count for remote queries is not available here.
const resultCount1 = h1.t === 'local'
? h1.completedQuery?.resultCount ?? -1
: -1;
: h1.resultCount ?? -1;
const resultCount2 = h2.t === 'local'
? h2.completedQuery?.resultCount ?? -1
: -1;
: h2.resultCount ?? -1;
switch (this.sortOrder) {
case SortOrder.NameAsc:
@@ -253,7 +266,10 @@ export class HistoryTreeDataProvider extends DisposableObject {
}
setCurrentItem(item?: QueryHistoryInfo) {
this.current = item;
if (item !== this.current) {
this.current = item;
this._onDidChangeCurrentQueryItem.fire(item);
}
}
remove(item: QueryHistoryInfo) {
@@ -279,7 +295,7 @@ export class HistoryTreeDataProvider extends DisposableObject {
set allHistory(history: QueryHistoryInfo[]) {
this.history = history;
this.current = history[0];
this.setCurrentItem(history[0]);
this.refresh();
}
@@ -306,21 +322,18 @@ export class QueryHistoryManager extends DisposableObject {
queryHistoryScrubber: Disposable | undefined;
private queryMetadataStorageLocation;
private readonly _onDidAddQueryItem = super.push(new EventEmitter<QueryHistoryInfo>());
readonly onDidAddQueryItem: Event<QueryHistoryInfo> = this
._onDidAddQueryItem.event;
private readonly _onDidChangeCurrentQueryItem = super.push(new EventEmitter<QueryHistoryInfo | undefined>());
readonly onDidChangeCurrentQueryItem = this._onDidChangeCurrentQueryItem.event;
private readonly _onDidRemoveQueryItem = super.push(new EventEmitter<QueryHistoryInfo>());
readonly onDidRemoveQueryItem: Event<QueryHistoryInfo> = this
._onDidRemoveQueryItem.event;
private readonly _onWillOpenQueryItem = super.push(new EventEmitter<QueryHistoryInfo>());
readonly onWillOpenQueryItem: Event<QueryHistoryInfo> = this
._onWillOpenQueryItem.event;
private readonly _onDidCompleteQuery = super.push(new EventEmitter<LocalQueryInfo>());
readonly onDidCompleteQuery = this._onDidCompleteQuery.event;
constructor(
private readonly qs: QueryServerClient,
private readonly dbm: DatabaseManager,
private readonly localQueriesInterfaceManager: InterfaceManager,
private readonly remoteQueriesManager: RemoteQueriesManager,
private readonly evalLogViewer: EvalLogViewer,
private readonly queryStorageDir: string,
private readonly ctx: ExtensionContext,
private readonly queryHistoryConfigListener: QueryHistoryConfig,
@@ -347,6 +360,11 @@ export class QueryHistoryManager extends DisposableObject {
canSelectMany: true,
}));
// Forward any change of current history item from the tree data.
this.push(this.treeDataProvider.onDidChangeCurrentQueryItem((item) => {
this._onDidChangeCurrentQueryItem.fire(item);
}));
// Lazily update the tree view selection due to limitations of TreeView API (see
// `updateTreeViewSelectionIfVisible` doc for details)
this.push(
@@ -438,6 +456,12 @@ export class QueryHistoryManager extends DisposableObject {
this.handleShowEvalLogSummary.bind(this)
)
);
this.push(
commandRunner(
'codeQLQueryHistory.showEvalLogViewer',
this.handleShowEvalLogViewer.bind(this)
)
);
this.push(
commandRunner(
'codeQLQueryHistory.cancel',
@@ -496,6 +520,12 @@ export class QueryHistoryManager extends DisposableObject {
}
)
);
this.push(
commandRunner(
'codeQLQueryHistory.copyRepoList',
this.handleCopyRepoList.bind(this)
)
);
// There are two configuration items that affect the query history:
// 1. The ttl for query history items.
@@ -504,7 +534,7 @@ export class QueryHistoryManager extends DisposableObject {
this.push(
queryHistoryConfigListener.onDidChangeConfiguration(() => {
this.treeDataProvider.refresh();
this.registerQueryHistoryScrubber(queryHistoryConfigListener, ctx);
this.registerQueryHistoryScrubber(queryHistoryConfigListener, this, ctx);
})
);
@@ -523,7 +553,13 @@ export class QueryHistoryManager extends DisposableObject {
},
}));
this.registerQueryHistoryScrubber(queryHistoryConfigListener, ctx);
this.registerQueryHistoryScrubber(queryHistoryConfigListener, this, ctx);
this.registerToRemoteQueriesEvents();
}
public completeQuery(info: LocalQueryInfo, results: QueryWithResults): void {
info.completeThisQuery(results);
this._onDidCompleteQuery.fire(info);
}
private getCredentials() {
@@ -533,7 +569,7 @@ export class QueryHistoryManager extends DisposableObject {
/**
* Register and create the history scrubber.
*/
private registerQueryHistoryScrubber(queryHistoryConfigListener: QueryHistoryConfig, ctx: ExtensionContext) {
private registerQueryHistoryScrubber(queryHistoryConfigListener: QueryHistoryConfig, qhm: QueryHistoryManager, ctx: ExtensionContext) {
this.queryHistoryScrubber?.dispose();
// Every hour check if we need to re-run the query history scrubber.
this.queryHistoryScrubber = this.push(
@@ -542,17 +578,61 @@ export class QueryHistoryManager extends DisposableObject {
TWO_HOURS_IN_MS,
queryHistoryConfigListener.ttlInMillis,
this.queryStorageDir,
qhm,
ctx
)
);
}
private registerToRemoteQueriesEvents() {
const queryAddedSubscription = this.remoteQueriesManager.onRemoteQueryAdded(async (event) => {
this.addQuery({
t: 'remote',
status: QueryStatus.InProgress,
completed: false,
queryId: event.queryId,
remoteQuery: event.query,
});
await this.refreshTreeView();
});
const queryRemovedSubscription = this.remoteQueriesManager.onRemoteQueryRemoved(async (event) => {
const item = this.treeDataProvider.allHistory.find(i => i.t === 'remote' && i.queryId === event.queryId);
if (item) {
await this.removeRemoteQuery(item as RemoteQueryHistoryItem);
}
});
const queryStatusUpdateSubscription = this.remoteQueriesManager.onRemoteQueryStatusUpdate(async (event) => {
const item = this.treeDataProvider.allHistory.find(i => i.t === 'remote' && i.queryId === event.queryId);
if (item) {
const remoteQueryHistoryItem = item as RemoteQueryHistoryItem;
remoteQueryHistoryItem.status = event.status;
remoteQueryHistoryItem.failureReason = event.failureReason;
remoteQueryHistoryItem.resultCount = event.resultCount;
if (event.status === QueryStatus.Completed) {
remoteQueryHistoryItem.completed = true;
}
await this.refreshTreeView();
} else {
void logger.log('Variant analysis status update event received for unknown variant analysis');
}
});
this.push(queryAddedSubscription);
this.push(queryRemovedSubscription);
this.push(queryStatusUpdateSubscription);
}
async readQueryHistory(): Promise<void> {
void logger.log(`Reading cached query history from '${this.queryMetadataStorageLocation}'.`);
const history = await slurpQueryHistory(this.queryMetadataStorageLocation);
this.treeDataProvider.allHistory = history;
this.treeDataProvider.allHistory.forEach((item) => {
this._onDidAddQueryItem.fire(item);
this.treeDataProvider.allHistory.forEach(async (item) => {
if (item.t === 'remote') {
await this.remoteQueriesManager.rehydrateRemoteQuery(item.queryId, item.remoteQuery, item.status);
}
});
}
@@ -600,6 +680,15 @@ export class QueryHistoryManager extends DisposableObject {
return this.treeDataProvider.getCurrent();
}
async removeDeletedQueries() {
await Promise.all(this.treeDataProvider.allHistory.map(async (item) => {
if (item.t == 'local' && item.completedQuery && !(await fs.pathExists(item.completedQuery?.query.querySaveDir))) {
this.treeDataProvider.remove(item);
item.completedQuery?.dispose();
}
}));
}
async handleRemoveHistoryItem(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[] = []
@@ -618,26 +707,30 @@ export class QueryHistoryManager extends DisposableObject {
await item.completedQuery?.query.deleteQuery();
}
} else {
// Remote queries can be removed locally, but not remotely.
// The user must cancel the query on GitHub Actions explicitly.
this.treeDataProvider.remove(item);
void logger.log(`Deleted ${this.labelProvider.getLabel(item)}.`);
if (item.status === QueryStatus.InProgress) {
void logger.log('The variant analysis is still running on GitHub Actions. To cancel there, you must go to the workflow run in your browser.');
}
this._onDidRemoveQueryItem.fire(item);
await this.removeRemoteQuery(item);
}
}));
await this.writeQueryHistory();
const current = this.treeDataProvider.getCurrent();
if (current !== undefined) {
await this.treeView.reveal(current, { select: true });
this._onWillOpenQueryItem.fire(current);
await this.openQueryResults(current);
}
}
private async removeRemoteQuery(item: RemoteQueryHistoryItem): Promise<void> {
// Remote queries can be removed locally, but not remotely.
// The user must cancel the query on GitHub Actions explicitly.
this.treeDataProvider.remove(item);
void logger.log(`Deleted ${this.labelProvider.getLabel(item)}.`);
if (item.status === QueryStatus.InProgress) {
void logger.log('The variant analysis is still running on GitHub Actions. To cancel there, you must go to the workflow run in your browser.');
}
await this.remoteQueriesManager.removeRemoteQuery(item.queryId);
}
async handleSortByName() {
if (this.treeDataProvider.sortOrder === SortOrder.NameAsc) {
this.treeDataProvider.sortOrder = SortOrder.NameDesc;
@@ -738,7 +831,7 @@ export class QueryHistoryManager extends DisposableObject {
} else {
// show results on single click only if query is completed successfully.
if (finalSingleItem.status === QueryStatus.Completed) {
await this._onWillOpenQueryItem.fire(finalSingleItem);
await this.openQueryResults(finalSingleItem);
}
}
}
@@ -811,16 +904,16 @@ export class QueryHistoryManager extends DisposableObject {
}
}
private warnNoEvalLog() {
void showAndLogWarningMessage(`No evaluator log is available for this run. Perhaps it failed before evaluation, or you are running with a version of CodeQL before ' + ${CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG}?`);
}
private warnNoEvalLogSummary() {
void showAndLogWarningMessage(`Evaluator log summary and evaluator log are not available for this run. Perhaps they failed before evaluation, or you are running with a version of CodeQL before ${CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG}?`);
private warnNoEvalLogs() {
void showAndLogWarningMessage(`Evaluator log, summary, and viewer are not available for this run. Perhaps it failed before evaluation, or you are running with a version of CodeQL before ' + ${CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG}?`);
}
private warnInProgressEvalLogSummary() {
void showAndLogWarningMessage('The evaluator log summary is still being generated. Please try again later. The summary generation process is tracked in the "CodeQL Extension Log" view.');
void showAndLogWarningMessage('The evaluator log summary is still being generated for this run. Please try again later. The summary generation process is tracked in the "CodeQL Extension Log" view.');
}
private warnInProgressEvalLogViewer() {
void showAndLogWarningMessage('The viewer\'s data is still being generated for this run. Please try again or re-run the query.');
}
async handleShowEvalLog(
@@ -837,7 +930,7 @@ export class QueryHistoryManager extends DisposableObject {
if (finalSingleItem.evalLogLocation) {
await this.tryOpenExternalFile(finalSingleItem.evalLogLocation);
} else {
this.warnNoEvalLog();
this.warnNoEvalLogs();
}
}
@@ -854,15 +947,41 @@ export class QueryHistoryManager extends DisposableObject {
if (finalSingleItem.evalLogSummaryLocation) {
await this.tryOpenExternalFile(finalSingleItem.evalLogSummaryLocation);
}
return;
}
// Summary log file doesn't exist.
else {
if (finalSingleItem.evalLogLocation && fs.pathExists(finalSingleItem.evalLogLocation)) {
// If raw log does exist, then the summary log is still being generated.
this.warnInProgressEvalLogSummary();
} else {
this.warnNoEvalLogSummary();
}
if (finalSingleItem.evalLogLocation && await fs.pathExists(finalSingleItem.evalLogLocation)) {
// If raw log does exist, then the summary log is still being generated.
this.warnInProgressEvalLogSummary();
} else {
this.warnNoEvalLogs();
}
}
async handleShowEvalLogViewer(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[],
) {
const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
// Only applicable to an individual local query
if (!this.assertSingleQuery(finalMultiSelect) || !finalSingleItem || finalSingleItem.t !== 'local') {
return;
}
// If the JSON summary file location wasn't saved, display error
if (finalSingleItem.jsonEvalLogSummaryLocation == undefined) {
this.warnInProgressEvalLogViewer();
return;
}
// TODO(angelapwen): Stream the file in.
try {
const evalLogData: EvalLogData[] = await parseViewerData(finalSingleItem.jsonEvalLogSummaryLocation);
const evalLogTreeBuilder = new EvalLogTreeBuilder(finalSingleItem.getQueryName(), evalLogData);
this.evalLogViewer.updateRoots(await evalLogTreeBuilder.getRoots());
} catch (e) {
throw new Error(`Could not read evaluator log summary JSON file to generate viewer data at ${finalSingleItem.jsonEvalLogSummaryLocation}.`);
}
}
@@ -870,8 +989,6 @@ export class QueryHistoryManager extends DisposableObject {
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[]
) {
// Local queries only
// In the future, we may support cancelling remote queries, but this is not a short term plan.
const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
const selected = finalMultiSelect || [finalSingleItem];
@@ -955,11 +1072,11 @@ export class QueryHistoryManager extends DisposableObject {
void this.tryOpenExternalFile(query.csvPath);
return;
}
await query.exportCsvResults(this.qs, query.csvPath, () => {
if (await query.exportCsvResults(this.qs, query.csvPath)) {
void this.tryOpenExternalFile(
query.csvPath
);
});
}
}
async handleViewCsvAlerts(
@@ -1013,6 +1130,20 @@ export class QueryHistoryManager extends DisposableObject {
);
}
async handleCopyRepoList(
singleItem: QueryHistoryInfo,
multiSelect: QueryHistoryInfo[],
) {
const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
// Remote queries only
if (!this.assertSingleQuery(finalMultiSelect) || !finalSingleItem || finalSingleItem.t !== 'remote') {
return;
}
await commands.executeCommand('codeQL.copyRepoList', finalSingleItem.queryId);
}
async getQueryText(item: QueryHistoryInfo): Promise<string> {
return item.t === 'local'
? item.initialInfo.queryText
@@ -1026,7 +1157,6 @@ export class QueryHistoryManager extends DisposableObject {
addQuery(item: QueryHistoryInfo) {
this.treeDataProvider.pushQuery(item);
this.updateTreeViewSelectionIfVisible();
this._onDidAddQueryItem.fire(item);
}
/**
@@ -1227,4 +1357,13 @@ the file in the file explorer and dragging it into the workspace.`
this.treeDataProvider.refresh();
await this.writeQueryHistory();
}
private async openQueryResults(item: QueryHistoryInfo) {
if (item.t === 'local') {
await this.localQueriesInterfaceManager.showResults(item as CompletedLocalQueryInfo, WebviewReveal.Forced, false);
}
else if (item.t === 'remote') {
await this.remoteQueriesManager.openRemoteQueryResults(item.queryId);
}
}
}

View File

@@ -217,6 +217,8 @@ export class LocalQueryInfo {
public completedQuery: CompletedQueryInfo | undefined;
public evalLogLocation: string | undefined;
public evalLogSummaryLocation: string | undefined;
public jsonEvalLogSummaryLocation: string | undefined;
public evalLogSummarySymbolsLocation: string | undefined;
/**
* Note that in the {@link slurpQueryHistory} method, we create a FullQueryInfo instance
@@ -281,7 +283,7 @@ export class LocalQueryInfo {
return !!this.completedQuery;
}
completeThisQuery(info: QueryWithResults) {
completeThisQuery(info: QueryWithResults): void {
this.completedQuery = new CompletedQueryInfo(info);
// dispose of the cancellation token source and also ensure the source is not serialized as JSON

View File

@@ -4,6 +4,7 @@ import * as path from 'path';
import { showAndLogErrorMessage } from './helpers';
import { asyncFilter, getErrorMessage, getErrorStack } from './pure/helpers-pure';
import { CompletedQueryInfo, LocalQueryInfo, QueryHistoryInfo } from './query-results';
import { QueryStatus } from './query-status';
import { QueryEvaluationInfo } from './run-queries';
export async function slurpQueryHistory(fsPath: string): Promise<QueryHistoryInfo[]> {
@@ -39,7 +40,12 @@ export async function slurpQueryHistory(fsPath: string): Promise<QueryHistoryInf
q.completedQuery.dispose = () => { /**/ };
}
} else if (q.t === 'remote') {
// noop
// A bug was introduced that didn't set the completed flag in query history
// items. The following code makes sure that the flag is set in order to
// "patch" older query history items.
if (q.status === QueryStatus.Completed) {
q.completed = true;
}
}
return q;
});

View File

@@ -267,6 +267,14 @@ export function findQueryEvalLogSummaryFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log.summary');
}
export function findJsonQueryEvalLogSummaryFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log.summary.jsonl');
}
export function findQueryEvalLogSummarySymbolsFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log.summary.symbols.json');
}
export function findQueryEvalLogEndSummaryFile(resultPath: string): string {
return path.join(resultPath, 'evaluator-log-end.summary');
}
}

View File

@@ -121,15 +121,22 @@ export async function displayQuickQuery(
const quickQueryQlpackYaml: any = {
name: 'vscode/quick-query',
version: '1.0.0',
libraryPathDependencies: [qlpack]
dependencies: {
[qlpack]: '*'
}
};
await fs.writeFile(qlPackFile, QLPACK_FILE_HEADER + yaml.safeDump(quickQueryQlpackYaml), 'utf8');
await fs.writeFile(qlPackFile, QLPACK_FILE_HEADER + yaml.dump(quickQueryQlpackYaml), 'utf8');
}
if (shouldRewrite || !(await fs.pathExists(qlFile))) {
await fs.writeFile(qlFile, getInitialQueryContents(dbItem.language, dbscheme), 'utf8');
}
if (shouldRewrite) {
await cliServer.clearCache();
await cliServer.packInstall(queriesDir, true);
}
await Window.showTextDocument(await workspace.openTextDocument(qlFile));
} catch (e) {
if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
@@ -144,6 +151,6 @@ async function checkShouldRewrite(qlPackFile: string, newDependency: string) {
if (!(await fs.pathExists(qlPackFile))) {
return true;
}
const qlPackContents: any = yaml.safeLoad(await fs.readFile(qlPackFile, 'utf8'));
return qlPackContents.libraryPathDependencies?.[0] !== newDependency;
const qlPackContents: any = yaml.load(await fs.readFile(qlPackFile, 'utf8'));
return !qlPackContents.dependencies?.[newDependency];
}

View File

@@ -116,7 +116,10 @@ export class AnalysesResultsManager {
const analysisResults: AnalysisResults = {
nwo: analysis.nwo,
status: 'InProgress',
interpretedResults: []
interpretedResults: [],
resultCount: analysis.resultCount,
starCount: analysis.starCount,
lastUpdated: analysis.lastUpdated,
};
const queryId = analysis.downloadLink.queryId;
const resultsForQuery = this.internalGetAnalysesResults(queryId);
@@ -145,7 +148,7 @@ export class AnalysesResultsManager {
status: 'Completed'
};
} else if (fileExtension === '.bqrs') {
const queryResults = await this.readBqrsResults(artifactPath, fileLinkPrefix);
const queryResults = await this.readBqrsResults(artifactPath, fileLinkPrefix, analysis.sourceLocationPrefix);
newAnaysisResults = {
...analysisResults,
rawResults: queryResults,
@@ -177,8 +180,8 @@ export class AnalysesResultsManager {
return await fs.pathExists(createDownloadPath(this.storagePath, analysis.downloadLink));
}
private async readBqrsResults(filePath: string, fileLinkPrefix: string): Promise<AnalysisRawResults> {
return await extractRawResults(this.cliServer, this.logger, filePath, fileLinkPrefix);
private async readBqrsResults(filePath: string, fileLinkPrefix: string, sourceLocationPrefix: string): Promise<AnalysisRawResults> {
return await extractRawResults(this.cliServer, this.logger, filePath, fileLinkPrefix, sourceLocationPrefix);
}
private async readSarifResults(filePath: string, fileLinkPrefix: string): Promise<AnalysisAlert[]> {

View File

@@ -9,6 +9,7 @@ export async function extractRawResults(
logger: Logger,
filePath: string,
fileLinkPrefix: string,
sourceLocationPrefix: string
): Promise<AnalysisRawResults> {
const bqrsInfo = await cliServer.bqrsInfo(filePath);
const resultSets = bqrsInfo['result-sets'];
@@ -31,5 +32,5 @@ export async function extractRawResults(
const capped = !!chunk.next;
return { schema, resultSet, fileLinkPrefix, capped };
return { schema, resultSet, fileLinkPrefix, sourceLocationPrefix, capped };
}

View File

@@ -4,14 +4,17 @@ import * as fs from 'fs-extra';
import { window, commands, Uri, ExtensionContext, QuickPickItem, workspace, ViewColumn } from 'vscode';
import { Credentials } from '../authentication';
import { UserCancellationException } from '../commandRunner';
import { showInformationMessageWithAction } from '../helpers';
import {
showInformationMessageWithAction,
pluralize
} from '../helpers';
import { logger } from '../logging';
import { QueryHistoryManager } from '../query-history';
import { createGist } from './gh-actions-api-client';
import { RemoteQueriesManager } from './remote-queries-manager';
import { generateMarkdown } from './remote-queries-markdown-generation';
import { RemoteQuery } from './remote-query';
import { AnalysisResults } from './shared/analysis-result';
import { AnalysisResults, sumAnalysesResults } from './shared/analysis-result';
/**
* Exports the results of the currently-selected remote query.
@@ -74,13 +77,13 @@ async function determineExportFormat(
/**
* Converts the results of a remote query to markdown and uploads the files as a secret gist.
*/
async function exportResultsToGist(
export async function exportResultsToGist(
ctx: ExtensionContext,
query: RemoteQuery,
analysesResults: AnalysisResults[]
): Promise<void> {
const credentials = await Credentials.initialize(ctx);
const description = 'CodeQL Variant Analysis Results';
const description = buildGistDescription(query, analysesResults);
const markdownFiles = generateMarkdown(query, analysesResults, 'gist');
// Convert markdownFiles to the appropriate format for uploading to gist
const gistFiles = markdownFiles.reduce((acc, cur) => {
@@ -100,6 +103,17 @@ async function exportResultsToGist(
}
}
/**
* Builds Gist description
* Ex: Empty Block (Go) x results (y repositories)
*/
const buildGistDescription = (query: RemoteQuery, analysesResults: AnalysisResults[]) => {
const resultCount = sumAnalysesResults(analysesResults);
const resultLabel = pluralize(resultCount, 'result', 'results');
const repositoryLabel = query.repositoryCount ? `(${pluralize(query.repositoryCount, 'repository', 'repositories')})` : '';
return `${query.queryName} (${query.language}) ${resultLabel} ${repositoryLabel}`;
};
/**
* Converts the results of a remote query to markdown and saves the files locally
* in the query directory (where query results and metadata are also saved).

View File

@@ -1,13 +1,14 @@
import * as unzipper from 'unzipper';
import * as path from 'path';
import * as fs from 'fs-extra';
import { showAndLogWarningMessage, tmpDir } from '../helpers';
import { showAndLogErrorMessage, showAndLogWarningMessage, tmpDir } from '../helpers';
import { Credentials } from '../authentication';
import { logger } from '../logging';
import { RemoteQueryWorkflowResult } from './remote-query-workflow-result';
import { DownloadLink, createDownloadPath } from './download-link';
import { RemoteQuery } from './remote-query';
import { RemoteQueryFailureIndexItem, RemoteQueryResultIndex, RemoteQuerySuccessIndexItem } from './remote-query-result-index';
import { getErrorMessage } from '../pure/helpers-pure';
interface ApiSuccessIndexItem {
nwo: string;
@@ -16,6 +17,7 @@ interface ApiSuccessIndexItem {
results_count: number;
bqrs_file_size: number;
sarif_file_size?: number;
source_location_prefix: string;
}
interface ApiFailureIndexItem {
@@ -58,7 +60,8 @@ export async function getRemoteQueryIndex(
sha: item.sha,
resultCount: item.results_count,
bqrsFileSize: item.bqrs_file_size,
sarifFileSize: item.sarif_file_size
sarifFileSize: item.sarif_file_size,
sourceLocationPrefix: item.source_location_prefix
} as RemoteQuerySuccessIndexItem;
});
@@ -332,3 +335,79 @@ export async function createGist(
}
return response.data.html_url;
}
const repositoriesMetadataQuery = `query Stars($repos: String!, $pageSize: Int!, $cursor: String) {
search(
query: $repos
type: REPOSITORY
first: $pageSize
after: $cursor
) {
edges {
node {
... on Repository {
name
owner {
login
}
stargazerCount
updatedAt
}
}
cursor
}
}
}`;
type RepositoriesMetadataQueryResponse = {
search: {
edges: {
cursor: string;
node: {
name: string;
owner: {
login: string;
};
stargazerCount: number;
updatedAt: string; // Actually a ISO Date string
}
}[]
}
};
export type RepositoriesMetadata = Record<string, { starCount: number, lastUpdated: number }>
export async function getRepositoriesMetadata(credentials: Credentials, nwos: string[], pageSize = 100): Promise<RepositoriesMetadata> {
const octokit = await credentials.getOctokit();
const repos = `repo:${nwos.join(' repo:')} fork:true`;
let cursor = null;
const metadata: RepositoriesMetadata = {};
try {
do {
const response: RepositoriesMetadataQueryResponse = await octokit.graphql({
query: repositoriesMetadataQuery,
repos,
pageSize,
cursor
});
cursor = response.search.edges.length === pageSize ? response.search.edges[pageSize - 1].cursor : null;
for (const edge of response.search.edges) {
const node = edge.node;
const owner = node.owner.login;
const name = node.name;
const starCount = node.stargazerCount;
// lastUpdated is always negative since it happened in the past.
const lastUpdated = new Date(node.updatedAt).getTime() - Date.now();
metadata[`${owner}/${name}`] = {
starCount, lastUpdated
};
}
} while (cursor);
} catch (e) {
void showAndLogErrorMessage(`Error retrieving repository metadata for variant analysis: ${getErrorMessage(e)}`);
}
return metadata;
}

View File

@@ -1,11 +1,10 @@
import {
WebviewPanel,
ExtensionContext,
window as Window,
ViewColumn,
Uri,
workspace,
commands
commands,
} from 'vscode';
import * as path from 'path';
@@ -16,29 +15,34 @@ import {
RemoteQueryDownloadAllAnalysesResultsMessage
} from '../pure/interface-types';
import { Logger } from '../logging';
import { getHtmlForWebview } from '../interface-utils';
import { assertNever } from '../pure/helpers-pure';
import { AnalysisSummary, RemoteQueryResult } from './remote-query-result';
import {
AnalysisSummary,
RemoteQueryResult,
sumAnalysisSummariesResults
} from './remote-query-result';
import { RemoteQuery } from './remote-query';
import { RemoteQueryResult as RemoteQueryResultViewModel } from './shared/remote-query-result';
import { AnalysisSummary as AnalysisResultViewModel } from './shared/remote-query-result';
import {
AnalysisSummary as AnalysisResultViewModel,
RemoteQueryResult as RemoteQueryResultViewModel
} from './shared/remote-query-result';
import { showAndLogWarningMessage } from '../helpers';
import { URLSearchParams } from 'url';
import { SHOW_QUERY_TEXT_MSG } from '../query-history';
import { AnalysesResultsManager } from './analyses-results-manager';
import { AnalysisResults } from './shared/analysis-result';
import { humanizeUnit } from '../pure/time';
import { AbstractInterfaceManager, InterfacePanelConfig } from '../abstract-interface-manager';
export class RemoteQueriesInterfaceManager {
private panel: WebviewPanel | undefined;
private panelLoaded = false;
export class RemoteQueriesInterfaceManager extends AbstractInterfaceManager<ToRemoteQueriesMessage, FromRemoteQueriesMessage> {
private currentQueryId: string | undefined;
private panelLoadedCallBacks: (() => void)[] = [];
constructor(
private readonly ctx: ExtensionContext,
ctx: ExtensionContext,
private readonly logger: Logger,
private readonly analysesResultsManager: AnalysesResultsManager
) {
super(ctx);
this.panelLoadedCallBacks.push(() => {
void logger.log('Variant analysis results view loaded');
});
@@ -72,13 +76,14 @@ export class RemoteQueriesInterfaceManager {
*/
private buildViewModel(query: RemoteQuery, queryResult: RemoteQueryResult): RemoteQueryResultViewModel {
const queryFileName = path.basename(query.queryFilePath);
const totalResultCount = queryResult.analysisSummaries.reduce((acc, cur) => acc + cur.resultCount, 0);
const totalResultCount = sumAnalysisSummariesResults(queryResult.analysisSummaries);
const executionDuration = this.getDuration(queryResult.executionEndTime, query.executionStartTime);
const analysisSummaries = this.buildAnalysisSummaries(queryResult.analysisSummaries);
const totalRepositoryCount = queryResult.analysisSummaries.length;
const affectedRepositories = queryResult.analysisSummaries.filter(r => r.resultCount > 0);
return {
queryId: queryResult.queryId,
queryTitle: query.queryName,
queryFileName: queryFileName,
queryFilePath: query.queryFilePath,
@@ -95,69 +100,56 @@ export class RemoteQueriesInterfaceManager {
};
}
getPanel(): WebviewPanel {
if (this.panel == undefined) {
const { ctx } = this;
const panel = (this.panel = Window.createWebviewPanel(
'remoteQueriesView',
'CodeQL Query Results',
{ viewColumn: ViewColumn.Active, preserveFocus: true },
{
enableScripts: true,
enableFindWidget: true,
retainContextWhenHidden: true,
localResourceRoots: [
Uri.file(this.analysesResultsManager.storagePath),
Uri.file(path.join(this.ctx.extensionPath, 'out')),
],
}
));
this.panel.onDidDispose(
() => {
this.panel = undefined;
this.currentQueryId = undefined;
},
null,
ctx.subscriptions
);
const scriptPathOnDisk = Uri.file(
ctx.asAbsolutePath('out/remoteQueriesView.js')
);
const baseStylesheetUriOnDisk = Uri.file(
ctx.asAbsolutePath('out/remote-queries/view/baseStyles.css')
);
const stylesheetPathOnDisk = Uri.file(
ctx.asAbsolutePath('out/remote-queries/view/remoteQueries.css')
);
panel.webview.html = getHtmlForWebview(
panel.webview,
scriptPathOnDisk,
[baseStylesheetUriOnDisk, stylesheetPathOnDisk],
true
);
ctx.subscriptions.push(
panel.webview.onDidReceiveMessage(
async (e) => this.handleMsgFromView(e),
undefined,
ctx.subscriptions
)
);
}
return this.panel;
protected getPanelConfig(): InterfacePanelConfig {
return {
viewId: 'remoteQueriesView',
title: 'CodeQL Query Results',
viewColumn: ViewColumn.Active,
preserveFocus: true,
view: 'remote-queries',
additionalOptions: {
localResourceRoots: [
Uri.file(this.analysesResultsManager.storagePath)
]
}
};
}
private waitForPanelLoaded(): Promise<void> {
return new Promise((resolve) => {
if (this.panelLoaded) {
resolve();
} else {
this.panelLoadedCallBacks.push(resolve);
}
});
protected onPanelDispose(): void {
this.currentQueryId = undefined;
}
protected async onMessage(msg: FromRemoteQueriesMessage): Promise<void> {
switch (msg.t) {
case 'remoteQueryLoaded':
this.onWebViewLoaded();
break;
case 'remoteQueryError':
void this.logger.log(
`Variant analysis error: ${msg.error}`
);
break;
case 'openFile':
await this.openFile(msg.filePath);
break;
case 'openVirtualFile':
await this.openVirtualFile(msg.queryText);
break;
case 'copyRepoList':
await commands.executeCommand('codeQL.copyRepoList', msg.queryId);
break;
case 'remoteQueryDownloadAnalysisResults':
await this.downloadAnalysisResults(msg);
break;
case 'remoteQueryDownloadAllAnalysesResults':
await this.downloadAllAnalysesResults(msg);
break;
case 'remoteQueryExportResults':
await commands.executeCommand('codeQL.exportVariantAnalysisResults');
break;
default:
assertNever(msg);
}
}
private async openFile(filePath: string) {
@@ -185,40 +177,6 @@ export class RemoteQueriesInterfaceManager {
}
}
private async handleMsgFromView(
msg: FromRemoteQueriesMessage
): Promise<void> {
switch (msg.t) {
case 'remoteQueryLoaded':
this.panelLoaded = true;
this.panelLoadedCallBacks.forEach((cb) => cb());
this.panelLoadedCallBacks = [];
break;
case 'remoteQueryError':
void this.logger.log(
`Variant analysis error: ${msg.error}`
);
break;
case 'openFile':
await this.openFile(msg.filePath);
break;
case 'openVirtualFile':
await this.openVirtualFile(msg.queryText);
break;
case 'remoteQueryDownloadAnalysisResults':
await this.downloadAnalysisResults(msg);
break;
case 'remoteQueryDownloadAllAnalysesResults':
await this.downloadAllAnalysesResults(msg);
break;
case 'remoteQueryExportResults':
await commands.executeCommand('codeQL.exportVariantAnalysisResults');
break;
default:
assertNever(msg);
}
}
private async downloadAnalysisResults(msg: RemoteQueryDownloadAnalysisResultsMessage): Promise<void> {
const queryId = this.currentQueryId;
await this.analysesResultsManager.downloadAnalysisResults(
@@ -243,29 +201,9 @@ export class RemoteQueriesInterfaceManager {
}
}
private postMessage(msg: ToRemoteQueriesMessage): Thenable<boolean> {
return this.getPanel().webview.postMessage(msg);
}
private getDuration(startTime: number, endTime: number): string {
const diffInMs = startTime - endTime;
return this.formatDuration(diffInMs);
}
private formatDuration(ms: number): string {
const seconds = ms / 1000;
const minutes = seconds / 60;
const hours = minutes / 60;
const days = hours / 24;
if (days > 1) {
return `${days.toFixed(2)} days`;
} else if (hours > 1) {
return `${hours.toFixed(2)} hours`;
} else if (minutes > 1) {
return `${minutes.toFixed(2)} minutes`;
} else {
return `${seconds.toFixed(2)} seconds`;
}
return humanizeUnit(diffInMs);
}
private formatDate = (millis: number): string => {
@@ -306,7 +244,10 @@ export class RemoteQueriesInterfaceManager {
databaseSha: analysisResult.databaseSha || 'HEAD',
resultCount: analysisResult.resultCount,
downloadLink: analysisResult.downloadLink,
fileSize: this.formatFileSize(analysisResult.fileSizeInBytes)
sourceLocationPrefix: analysisResult.sourceLocationPrefix,
fileSize: this.formatFileSize(analysisResult.fileSizeInBytes),
starCount: analysisResult.starCount,
lastUpdated: analysisResult.lastUpdated
}));
}
}

View File

@@ -1,7 +1,8 @@
import { CancellationToken, commands, ExtensionContext, Uri, window } from 'vscode';
import { CancellationToken, commands, EventEmitter, ExtensionContext, Uri, env, window } from 'vscode';
import { nanoid } from 'nanoid';
import * as path from 'path';
import * as fs from 'fs-extra';
import * as os from 'os';
import { Credentials } from '../authentication';
import { CodeQLCliServer } from '../cli';
@@ -12,24 +13,47 @@ import { runRemoteQuery } from './run-remote-query';
import { RemoteQueriesInterfaceManager } from './remote-queries-interface';
import { RemoteQuery } from './remote-query';
import { RemoteQueriesMonitor } from './remote-queries-monitor';
import { getRemoteQueryIndex } from './gh-actions-api-client';
import { getRemoteQueryIndex, getRepositoriesMetadata, RepositoriesMetadata } from './gh-actions-api-client';
import { RemoteQueryResultIndex } from './remote-query-result-index';
import { RemoteQueryResult } from './remote-query-result';
import { RemoteQueryResult, sumAnalysisSummariesResults } from './remote-query-result';
import { DownloadLink } from './download-link';
import { AnalysesResultsManager } from './analyses-results-manager';
import { assertNever } from '../pure/helpers-pure';
import { RemoteQueryHistoryItem } from './remote-query-history-item';
import { QueryHistoryManager } from '../query-history';
import { QueryStatus } from '../query-status';
import { DisposableObject } from '../pure/disposable-object';
import { QueryHistoryInfo } from '../query-results';
import { AnalysisResults } from './shared/analysis-result';
const autoDownloadMaxSize = 300 * 1024;
const autoDownloadMaxCount = 100;
const noop = () => { /* do nothing */ };
export interface NewQueryEvent {
queryId: string;
query: RemoteQuery
}
export interface RemovedQueryEvent {
queryId: string;
}
export interface UpdatedQueryStatusEvent {
queryId: string;
status: QueryStatus;
failureReason?: string;
repositoryCount?: number;
resultCount?: number;
}
export class RemoteQueriesManager extends DisposableObject {
public readonly onRemoteQueryAdded;
public readonly onRemoteQueryRemoved;
public readonly onRemoteQueryStatusUpdate;
private readonly remoteQueryAddedEventEmitter;
private readonly remoteQueryRemovedEventEmitter;
private readonly remoteQueryStatusUpdateEventEmitter;
private readonly remoteQueriesMonitor: RemoteQueriesMonitor;
private readonly analysesResultsManager: AnalysesResultsManager;
private readonly interfaceManager: RemoteQueriesInterfaceManager;
@@ -37,7 +61,6 @@ export class RemoteQueriesManager extends DisposableObject {
constructor(
private readonly ctx: ExtensionContext,
private readonly cliServer: CodeQLCliServer,
private readonly qhm: QueryHistoryManager,
private readonly storagePath: string,
logger: Logger,
) {
@@ -46,45 +69,45 @@ export class RemoteQueriesManager extends DisposableObject {
this.interfaceManager = new RemoteQueriesInterfaceManager(ctx, logger, this.analysesResultsManager);
this.remoteQueriesMonitor = new RemoteQueriesMonitor(ctx, logger);
// Handle events from the query history manager
this.push(this.qhm.onDidAddQueryItem(this.handleAddQueryItem.bind(this)));
this.push(this.qhm.onDidRemoveQueryItem(this.handleRemoveQueryItem.bind(this)));
this.push(this.qhm.onWillOpenQueryItem(this.handleOpenQueryItem.bind(this)));
this.remoteQueryAddedEventEmitter = this.push(new EventEmitter<NewQueryEvent>());
this.remoteQueryRemovedEventEmitter = this.push(new EventEmitter<RemovedQueryEvent>());
this.remoteQueryStatusUpdateEventEmitter = this.push(new EventEmitter<UpdatedQueryStatusEvent>());
this.onRemoteQueryAdded = this.remoteQueryAddedEventEmitter.event;
this.onRemoteQueryRemoved = this.remoteQueryRemovedEventEmitter.event;
this.onRemoteQueryStatusUpdate = this.remoteQueryStatusUpdateEventEmitter.event;
this.push(this.interfaceManager);
}
private async handleAddQueryItem(queryItem: QueryHistoryInfo) {
if (queryItem?.t === 'remote') {
if (!(await this.queryHistoryItemExists(queryItem))) {
// In this case, the query was deleted from disk, most likely because it was purged
// by another workspace. We should remove it from the history manager.
await this.qhm.handleRemoveHistoryItem(queryItem);
} else if (queryItem.status === QueryStatus.InProgress) {
// In this case, last time we checked, the query was still in progress.
// We need to setup the monitor to check for completion.
await commands.executeCommand('codeQL.monitorRemoteQuery', queryItem);
}
public async rehydrateRemoteQuery(queryId: string, query: RemoteQuery, status: QueryStatus) {
if (!(await this.queryRecordExists(queryId))) {
// In this case, the query was deleted from disk, most likely because it was purged
// by another workspace.
this.remoteQueryRemovedEventEmitter.fire({ queryId });
} else if (status === QueryStatus.InProgress) {
// In this case, last time we checked, the query was still in progress.
// We need to setup the monitor to check for completion.
await commands.executeCommand('codeQL.monitorRemoteQuery', queryId, query);
}
}
private async handleRemoveQueryItem(queryItem: QueryHistoryInfo) {
if (queryItem?.t === 'remote') {
this.analysesResultsManager.removeAnalysesResults(queryItem.queryId);
await this.removeStorageDirectory(queryItem);
}
public async removeRemoteQuery(queryId: string) {
this.analysesResultsManager.removeAnalysesResults(queryId);
await this.removeStorageDirectory(queryId);
}
private async handleOpenQueryItem(queryItem: QueryHistoryInfo) {
if (queryItem?.t === 'remote') {
try {
const remoteQueryResult = await this.retrieveJsonFile(queryItem, 'query-result.json') as RemoteQueryResult;
// open results in the background
void this.openResults(queryItem.remoteQuery, remoteQueryResult).then(
noop,
err => void showAndLogErrorMessage(err)
);
} catch (e) {
void showAndLogErrorMessage(`Could not open query results. ${e}`);
}
public async openRemoteQueryResults(queryId: string) {
try {
const remoteQuery = await this.retrieveJsonFile(queryId, 'query.json') as RemoteQuery;
const remoteQueryResult = await this.retrieveJsonFile(queryId, 'query-result.json') as RemoteQueryResult;
// Open results in the background
void this.openResults(remoteQuery, remoteQueryResult).then(
noop,
err => void showAndLogErrorMessage(err)
);
} catch (e) {
void showAndLogErrorMessage(`Could not open query results. ${e}`);
}
}
@@ -106,49 +129,40 @@ export class RemoteQueriesManager extends DisposableObject {
const query = querySubmission.query;
const queryId = this.createQueryId(query.queryName);
const queryHistoryItem: RemoteQueryHistoryItem = {
t: 'remote',
status: QueryStatus.InProgress,
completed: false,
queryId,
remoteQuery: query,
};
await this.prepareStorageDirectory(queryHistoryItem);
await this.storeJsonFile(queryHistoryItem, 'query.json', query);
await this.prepareStorageDirectory(queryId);
await this.storeJsonFile(queryId, 'query.json', query);
this.qhm.addQuery(queryHistoryItem);
await this.qhm.refreshTreeView();
this.remoteQueryAddedEventEmitter.fire({ queryId, query });
void commands.executeCommand('codeQL.monitorRemoteQuery', queryId, query);
}
}
public async monitorRemoteQuery(
queryItem: RemoteQueryHistoryItem,
queryId: string,
remoteQuery: RemoteQuery,
cancellationToken: CancellationToken
): Promise<void> {
const credentials = await Credentials.initialize(this.ctx);
const queryWorkflowResult = await this.remoteQueriesMonitor.monitorQuery(queryItem.remoteQuery, cancellationToken);
const queryWorkflowResult = await this.remoteQueriesMonitor.monitorQuery(remoteQuery, cancellationToken);
const executionEndTime = Date.now();
if (queryWorkflowResult.status === 'CompletedSuccessfully') {
await this.downloadAvailableResults(queryItem, credentials, executionEndTime);
await this.downloadAvailableResults(queryId, remoteQuery, credentials, executionEndTime);
} else if (queryWorkflowResult.status === 'CompletedUnsuccessfully') {
if (queryWorkflowResult.error?.includes('cancelled')) {
// workflow was cancelled on the server
queryItem.failureReason = 'Cancelled';
queryItem.status = QueryStatus.Failed;
await this.downloadAvailableResults(queryItem, credentials, executionEndTime);
// Workflow was cancelled on the server
this.remoteQueryStatusUpdateEventEmitter.fire({ queryId, status: QueryStatus.Failed, failureReason: 'Cancelled' });
await this.downloadAvailableResults(queryId, remoteQuery, credentials, executionEndTime);
void showAndLogInformationMessage('Variant analysis was cancelled');
} else {
queryItem.failureReason = queryWorkflowResult.error;
queryItem.status = QueryStatus.Failed;
this.remoteQueryStatusUpdateEventEmitter.fire({ queryId, status: QueryStatus.Failed, failureReason: queryWorkflowResult.error });
void showAndLogErrorMessage(`Variant analysis execution failed. Error: ${queryWorkflowResult.error}`);
}
} else if (queryWorkflowResult.status === 'Cancelled') {
queryItem.failureReason = 'Cancelled';
queryItem.status = QueryStatus.Failed;
await this.downloadAvailableResults(queryItem, credentials, executionEndTime);
this.remoteQueryStatusUpdateEventEmitter.fire({ queryId, status: QueryStatus.Failed, failureReason: 'Cancelled' });
await this.downloadAvailableResults(queryId, remoteQuery, credentials, executionEndTime);
void showAndLogInformationMessage('Variant analysis was cancelled');
} else if (queryWorkflowResult.status === 'InProgress') {
// Should not get here. Only including this to ensure `assertNever` uses proper type checking.
@@ -157,7 +171,6 @@ export class RemoteQueriesManager extends DisposableObject {
// Ensure all cases are covered
assertNever(queryWorkflowResult.status);
}
await this.qhm.refreshTreeView();
}
public async autoDownloadRemoteQueryResults(
@@ -171,6 +184,7 @@ export class RemoteQueriesManager extends DisposableObject {
nwo: a.nwo,
databaseSha: a.databaseSha,
resultCount: a.resultCount,
sourceLocationPrefix: a.sourceLocationPrefix,
downloadLink: a.downloadLink,
fileSize: String(a.fileSizeInBytes)
}));
@@ -181,18 +195,43 @@ export class RemoteQueriesManager extends DisposableObject {
results => this.interfaceManager.setAnalysisResults(results, queryResult.queryId));
}
private mapQueryResult(executionEndTime: number, resultIndex: RemoteQueryResultIndex, queryId: string): RemoteQueryResult {
public async copyRemoteQueryRepoListToClipboard(queryId: string) {
const queryResult = await this.getRemoteQueryResult(queryId);
const repos = queryResult.analysisSummaries
.filter(a => a.resultCount > 0)
.map(a => a.nwo);
if (repos.length > 0) {
const text = [
'"new-repo-list": [',
...repos.slice(0, -1).map(repo => ` "${repo}",`),
` "${repos[repos.length - 1]}"`,
']'
];
await env.clipboard.writeText(text.join(os.EOL));
}
}
private mapQueryResult(
executionEndTime: number,
resultIndex: RemoteQueryResultIndex,
queryId: string,
metadata: RepositoriesMetadata
): RemoteQueryResult {
const analysisSummaries = resultIndex.successes.map(item => ({
nwo: item.nwo,
databaseSha: item.sha || 'HEAD',
resultCount: item.resultCount,
sourceLocationPrefix: item.sourceLocationPrefix,
fileSizeInBytes: item.sarifFileSize ? item.sarifFileSize : item.bqrsFileSize,
starCount: metadata[item.nwo]?.starCount,
lastUpdated: metadata[item.nwo]?.lastUpdated,
downloadLink: {
id: item.artifactId.toString(),
urlPath: `${resultIndex.artifactsUrlPath}/${item.artifactId}`,
innerFilePath: item.sarifFileSize ? 'results.sarif' : 'results.bqrs',
queryId,
queryId
} as DownloadLink
}));
const analysisFailures = resultIndex.failures.map(item => ({
@@ -213,7 +252,7 @@ export class RemoteQueriesManager extends DisposableObject {
}
private async askToOpenResults(query: RemoteQuery, queryResult: RemoteQueryResult): Promise<void> {
const totalResultCount = queryResult.analysisSummaries.reduce((acc, cur) => acc + cur.resultCount, 0);
const totalResultCount = sumAnalysisSummariesResults(queryResult.analysisSummaries);
const totalRepoCount = queryResult.analysisSummaries.length;
const message = `Query "${query.queryName}" run on ${totalRepoCount} repositories and returned ${totalResultCount} results`;
@@ -230,7 +269,6 @@ export class RemoteQueriesManager extends DisposableObject {
*/
private createQueryId(queryName: string): string {
return `${queryName}-${nanoid()}`;
}
/**
@@ -239,29 +277,32 @@ export class RemoteQueriesManager extends DisposableObject {
* used by the query history manager to determine when the directory
* should be deleted.
*
* @param queryName The name of the query that was run.
*/
private async prepareStorageDirectory(queryHistoryItem: RemoteQueryHistoryItem): Promise<void> {
await createTimestampFile(path.join(this.storagePath, queryHistoryItem.queryId));
private async prepareStorageDirectory(queryId: string): Promise<void> {
await createTimestampFile(path.join(this.storagePath, queryId));
}
private async storeJsonFile<T>(queryHistoryItem: RemoteQueryHistoryItem, fileName: string, obj: T): Promise<void> {
const filePath = path.join(this.storagePath, queryHistoryItem.queryId, fileName);
private async getRemoteQueryResult(queryId: string): Promise<RemoteQueryResult> {
return await this.retrieveJsonFile<RemoteQueryResult>(queryId, 'query-result.json');
}
private async storeJsonFile<T>(queryId: string, fileName: string, obj: T): Promise<void> {
const filePath = path.join(this.storagePath, queryId, fileName);
await fs.writeFile(filePath, JSON.stringify(obj, null, 2), 'utf8');
}
private async retrieveJsonFile<T>(queryHistoryItem: RemoteQueryHistoryItem, fileName: string): Promise<T> {
const filePath = path.join(this.storagePath, queryHistoryItem.queryId, fileName);
private async retrieveJsonFile<T>(queryId: string, fileName: string): Promise<T> {
const filePath = path.join(this.storagePath, queryId, fileName);
return JSON.parse(await fs.readFile(filePath, 'utf8'));
}
private async removeStorageDirectory(queryItem: RemoteQueryHistoryItem): Promise<void> {
const filePath = path.join(this.storagePath, queryItem.queryId);
private async removeStorageDirectory(queryId: string): Promise<void> {
const filePath = path.join(this.storagePath, queryId);
await fs.remove(filePath);
}
private async queryHistoryItemExists(queryItem: RemoteQueryHistoryItem): Promise<boolean> {
const filePath = path.join(this.storagePath, queryItem.queryId);
private async queryRecordExists(queryId: string): Promise<boolean> {
const filePath = path.join(this.storagePath, queryId);
return await fs.pathExists(filePath);
}
@@ -270,40 +311,51 @@ export class RemoteQueriesManager extends DisposableObject {
* If so, set the query status to `Completed` and auto-download the results.
*/
private async downloadAvailableResults(
queryItem: RemoteQueryHistoryItem,
queryId: string,
remoteQuery: RemoteQuery,
credentials: Credentials,
executionEndTime: number
): Promise<void> {
const resultIndex = await getRemoteQueryIndex(credentials, queryItem.remoteQuery);
const resultIndex = await getRemoteQueryIndex(credentials, remoteQuery);
if (resultIndex) {
queryItem.completed = true;
queryItem.status = QueryStatus.Completed;
queryItem.failureReason = undefined;
const queryResult = this.mapQueryResult(executionEndTime, resultIndex, queryItem.queryId);
const metadata = await this.getRepositoriesMetadata(resultIndex, credentials);
const queryResult = this.mapQueryResult(executionEndTime, resultIndex, queryId, metadata);
const resultCount = sumAnalysisSummariesResults(queryResult.analysisSummaries);
this.remoteQueryStatusUpdateEventEmitter.fire({
queryId,
status: QueryStatus.Completed,
repositoryCount: queryResult.analysisSummaries.length,
resultCount
});
await this.storeJsonFile(queryItem, 'query-result.json', queryResult);
await this.storeJsonFile(queryId, 'query-result.json', queryResult);
// Kick off auto-download of results in the background.
void commands.executeCommand('codeQL.autoDownloadRemoteQueryResults', queryResult);
// Ask if the user wants to open the results in the background.
void this.askToOpenResults(queryItem.remoteQuery, queryResult).then(
void this.askToOpenResults(remoteQuery, queryResult).then(
noop,
err => {
void showAndLogErrorMessage(err);
}
);
} else {
const controllerRepo = `${queryItem.remoteQuery.controllerRepository.owner}/${queryItem.remoteQuery.controllerRepository.name}`;
const workflowRunUrl = `https://github.com/${controllerRepo}/actions/runs/${queryItem.remoteQuery.actionsWorkflowRunId}`;
const controllerRepo = `${remoteQuery.controllerRepository.owner}/${remoteQuery.controllerRepository.name}`;
const workflowRunUrl = `https://github.com/${controllerRepo}/actions/runs/${remoteQuery.actionsWorkflowRunId}`;
void showAndLogErrorMessage(
`There was an issue retrieving the result for the query [${queryItem.remoteQuery.queryName}](${workflowRunUrl}).`
`There was an issue retrieving the result for the query [${remoteQuery.queryName}](${workflowRunUrl}).`
);
queryItem.status = QueryStatus.Failed;
this.remoteQueryStatusUpdateEventEmitter.fire({ queryId, status: QueryStatus.Failed });
}
}
// Pulled from the analysis results manager, so that we can get access to
private async getRepositoriesMetadata(resultIndex: RemoteQueryResultIndex, credentials: Credentials) {
const nwos = resultIndex.successes.map(s => s.nwo);
return await getRepositoriesMetadata(credentials, nwos);
}
// Pulled from the analysis results manager, so that we can get access to
// analyses results from the "export results" command.
public getAnalysesResults(queryId: string): AnalysisResults[] {
return [...this.analysesResultsManager.getAnalysesResults(queryId)];

View File

@@ -138,7 +138,7 @@ function generateMarkdownForCodeSnippet(
const codeLines = codeSnippet.text
.split('\n')
.map((line, index) =>
highlightCodeLines(line, index + snippetStartLine, highlightedRegion)
highlightAndEscapeCodeLines(line, index + snippetStartLine, highlightedRegion)
);
// Make sure there are no extra newlines before or after the <code> block:
@@ -153,20 +153,25 @@ function generateMarkdownForCodeSnippet(
return lines;
}
function highlightCodeLines(
function highlightAndEscapeCodeLines(
line: string,
lineNumber: number,
highlightedRegion?: HighlightedRegion
): string {
if (!highlightedRegion || !shouldHighlightLine(lineNumber, highlightedRegion)) {
return line;
return escapeHtmlCharacters(line);
}
const partiallyHighlightedLine = parseHighlightedLine(
line,
lineNumber,
highlightedRegion
);
return `${partiallyHighlightedLine.plainSection1}<strong>${partiallyHighlightedLine.highlightedSection}</strong>${partiallyHighlightedLine.plainSection2}`;
const plainSection1 = escapeHtmlCharacters(partiallyHighlightedLine.plainSection1);
const highlightedSection = escapeHtmlCharacters(partiallyHighlightedLine.highlightedSection);
const plainSection2 = escapeHtmlCharacters(partiallyHighlightedLine.plainSection2);
return `${plainSection1}<strong>${highlightedSection}</strong>${plainSection2}`;
}
function generateMarkdownForAlertMessage(
@@ -238,7 +243,7 @@ function generateMarkdownForRawResults(
for (const row of analysisRawResults.resultSet.rows) {
const cells = row.map((cell) =>
generateMarkdownForRawTableCell(cell, analysisRawResults.fileLinkPrefix)
generateMarkdownForRawTableCell(cell, analysisRawResults.fileLinkPrefix, analysisRawResults.sourceLocationPrefix)
);
tableRows.push(`| ${cells.join(' | ')} |`);
}
@@ -247,7 +252,8 @@ function generateMarkdownForRawResults(
function generateMarkdownForRawTableCell(
value: CellValue,
fileLinkPrefix: string
fileLinkPrefix: string,
sourceLocationPrefix: string
) {
let cellValue: string;
switch (typeof value) {
@@ -258,8 +264,12 @@ function generateMarkdownForRawTableCell(
break;
case 'object':
{
const url = tryGetRemoteLocation(value.url, fileLinkPrefix);
cellValue = `[\`${convertNonPrintableChars(value.label)}\`](${url})`;
const url = tryGetRemoteLocation(value.url, fileLinkPrefix, sourceLocationPrefix);
if (url) {
cellValue = `[\`${convertNonPrintableChars(value.label)}\`](${url})`;
} else {
cellValue = `\`${convertNonPrintableChars(value.label)}\``;
}
}
break;
}
@@ -325,3 +335,10 @@ function createFileName(nwo: string) {
const [owner, repo] = nwo.split('/');
return `${owner}-${repo}`;
}
/**
* Escape characters that could be interpreted as HTML instead of raw code.
*/
function escapeHtmlCharacters(text: string): string {
return text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
}

View File

@@ -7,6 +7,7 @@ import { RemoteQuery } from './remote-query';
export interface RemoteQueryHistoryItem {
readonly t: 'remote';
failureReason?: string;
resultCount?: number;
status: QueryStatus;
completed: boolean;
readonly queryId: string,

View File

@@ -12,6 +12,7 @@ export interface RemoteQuerySuccessIndexItem {
resultCount: number;
bqrsFileSize: number;
sarifFileSize?: number;
sourceLocationPrefix: string;
}
export interface RemoteQueryFailureIndexItem {

View File

@@ -2,16 +2,26 @@ import { DownloadLink } from './download-link';
import { AnalysisFailure } from './shared/analysis-failure';
export interface RemoteQueryResult {
executionEndTime: number; // Can't use a Date here since it needs to be serialized and desserialized.
analysisSummaries: AnalysisSummary[];
analysisFailures: AnalysisFailure[];
queryId: string;
executionEndTime: number, // Can't use a Date here since it needs to be serialized and desserialized.
analysisSummaries: AnalysisSummary[],
analysisFailures: AnalysisFailure[],
queryId: string,
}
export interface AnalysisSummary {
nwo: string,
databaseSha: string,
resultCount: number,
sourceLocationPrefix: string,
downloadLink: DownloadLink,
fileSizeInBytes: number
fileSizeInBytes: number,
starCount?: number,
lastUpdated?: number,
}
/**
* Sums up the number of results for all repos queried via a remote query.
*/
export const sumAnalysisSummariesResults = (analysisSummaries: AnalysisSummary[]): number => {
return analysisSummaries.reduce((acc, cur) => acc + cur.resultCount, 0);
};

View File

@@ -8,4 +8,5 @@ export interface RemoteQuery {
controllerRepository: Repository;
executionStartTime: number; // Use number here since it needs to be serialized and desserialized.
actionsWorkflowRunId: number;
repositoryCount: number;
}

View File

@@ -1,18 +1,26 @@
import * as fs from 'fs-extra';
import { QuickPickItem, window } from 'vscode';
import { logger } from '../logging';
import { getRemoteRepositoryLists } from '../config';
import { REPO_REGEX } from '../pure/helpers-pure';
import { getRemoteRepositoryLists, getRemoteRepositoryListsPath } from '../config';
import { OWNER_REGEX, REPO_REGEX } from '../pure/helpers-pure';
import { UserCancellationException } from '../commandRunner';
export interface RepositorySelection {
repositories?: string[];
repositoryLists?: string[]
repositoryLists?: string[];
owners?: string[];
}
interface RepoListQuickPickItem extends QuickPickItem {
repositories?: string[];
repositoryList?: string;
useCustomRepository?: boolean;
useCustomRepo?: boolean;
useAllReposOfOwner?: boolean;
}
interface RepoList {
label: string;
repositories: string[];
}
/**
@@ -22,8 +30,9 @@ interface RepoListQuickPickItem extends QuickPickItem {
export async function getRepositorySelection(): Promise<RepositorySelection> {
const quickPickItems = [
createCustomRepoQuickPickItem(),
createAllReposOfOwnerQuickPickItem(),
...createSystemDefinedRepoListsQuickPickItems(),
...createUserDefinedRepoListsQuickPickItems(),
...(await createUserDefinedRepoListsQuickPickItems()),
];
const options = {
@@ -41,13 +50,28 @@ export async function getRepositorySelection(): Promise<RepositorySelection> {
} else if (quickpick?.repositoryList) {
void logger.log(`Selected repository list: ${quickpick.repositoryList}`);
return { repositoryLists: [quickpick.repositoryList] };
} else if (quickpick?.useCustomRepository) {
} else if (quickpick?.useCustomRepo) {
const customRepo = await getCustomRepo();
if (customRepo === undefined) {
// The user cancelled, do nothing.
throw new UserCancellationException('No repositories selected', true);
}
if (!customRepo || !REPO_REGEX.test(customRepo)) {
throw new UserCancellationException('Invalid repository format. Please enter a valid repository in the format <owner>/<repo> (e.g. github/codeql)');
}
void logger.log(`Entered repository: ${customRepo}`);
return { repositories: [customRepo] };
} else if (quickpick?.useAllReposOfOwner) {
const owner = await getOwner();
if (owner === undefined) {
// The user cancelled, do nothing.
throw new UserCancellationException('No repositories selected', true);
}
if (!owner || !OWNER_REGEX.test(owner)) {
throw new Error(`Invalid user or organization: ${owner}`);
}
void logger.log(`Entered owner: ${owner}`);
return { owners: [owner] };
} else {
// We don't need to display a warning pop-up in this case, since the user just escaped out of the operation.
// We set 'true' to make this a silent exception.
@@ -61,17 +85,11 @@ export async function getRepositorySelection(): Promise<RepositorySelection> {
* @returns A boolean flag indicating if the selection is valid or not.
*/
export function isValidSelection(repoSelection: RepositorySelection): boolean {
if (repoSelection.repositories === undefined && repoSelection.repositoryLists === undefined) {
return false;
}
if (repoSelection.repositories !== undefined && repoSelection.repositories.length === 0) {
return false;
}
if (repoSelection.repositoryLists?.length === 0) {
return false;
}
const repositories = repoSelection.repositories || [];
const repositoryLists = repoSelection.repositoryLists || [];
const owners = repoSelection.owners || [];
return true;
return (repositories.length > 0 || repositoryLists.length > 0 || owners.length > 0);
}
function createSystemDefinedRepoListsQuickPickItems(): RepoListQuickPickItem[] {
@@ -84,28 +102,97 @@ function createSystemDefinedRepoListsQuickPickItems(): RepoListQuickPickItem[] {
} as RepoListQuickPickItem));
}
function createUserDefinedRepoListsQuickPickItems(): RepoListQuickPickItem[] {
async function readExternalRepoLists(): Promise<RepoList[]> {
const repoLists: RepoList[] = [];
const path = getRemoteRepositoryListsPath();
if (!path) {
return repoLists;
}
await validateExternalRepoListsFile(path);
const json = await readExternalRepoListsJson(path);
for (const [repoListName, repositories] of Object.entries(json)) {
if (!Array.isArray(repositories)) {
throw Error('Invalid repository lists file. It should contain an array of repositories for each list.');
}
repoLists.push({
label: repoListName,
repositories
});
}
return repoLists;
}
async function validateExternalRepoListsFile(path: string): Promise<void> {
const pathExists = await fs.pathExists(path);
if (!pathExists) {
throw Error(`External repository lists file does not exist at ${path}`);
}
const pathStat = await fs.stat(path);
if (pathStat.isDirectory()) {
throw Error('External repository lists path should not point to a directory');
}
}
async function readExternalRepoListsJson(path: string): Promise<Record<string, unknown>> {
let json;
try {
const fileContents = await fs.readFile(path, 'utf8');
json = await JSON.parse(fileContents);
} catch (error) {
throw Error('Invalid repository lists file. It should contain valid JSON.');
}
if (Array.isArray(json)) {
throw Error('Invalid repository lists file. It should be an object mapping names to a list of repositories.');
}
return json;
}
function readRepoListsFromSettings(): RepoList[] {
const repoLists = getRemoteRepositoryLists();
if (!repoLists) {
return [];
}
return Object.entries(repoLists).map<RepoListQuickPickItem>(([label, repositories]) => (
return Object.entries(repoLists).map<RepoList>(([label, repositories]) => (
{
label, // the name of the repository list
repositories // the actual array of repositories
label,
repositories
}
));
}
async function createUserDefinedRepoListsQuickPickItems(): Promise<RepoListQuickPickItem[]> {
const repoListsFromSetings = readRepoListsFromSettings();
const repoListsFromExternalFile = await readExternalRepoLists();
return [...repoListsFromSetings, ...repoListsFromExternalFile];
}
function createCustomRepoQuickPickItem(): RepoListQuickPickItem {
return {
label: '$(edit) Enter a GitHub repository',
useCustomRepository: true,
useCustomRepo: true,
alwaysShow: true,
};
}
function createAllReposOfOwnerQuickPickItem(): RepoListQuickPickItem {
return {
label: '$(edit) Enter a GitHub user or organization',
useAllReposOfOwner: true,
alwaysShow: true
};
}
async function getCustomRepo(): Promise<string | undefined> {
return await window.showInputBox({
title: 'Enter a GitHub repository in the format <owner>/<repo> (e.g. github/codeql)',
@@ -114,3 +201,10 @@ async function getCustomRepo(): Promise<string | undefined> {
ignoreFocusOut: true,
});
}
async function getOwner(): Promise<string | undefined> {
return await window.showInputBox({
title: 'Enter a GitHub user or organization',
ignoreFocusOut: true
});
}

View File

@@ -11,6 +11,7 @@ import {
showAndLogErrorMessage,
showAndLogInformationMessage,
tryGetQueryMetadata,
pluralize,
tmpDir
} from '../helpers';
import { Credentials } from '../authentication';
@@ -38,8 +39,11 @@ interface QueriesResponse {
errors?: {
invalid_repositories?: string[],
repositories_without_database?: string[],
private_repositories?: string[],
cutoff_repositories?: string[],
cutoff_repositories_count?: number,
},
repositories_queried?: string[],
repositories_queried: string[],
}
/**
@@ -110,7 +114,7 @@ async function generateQueryPack(cliServer: cli.CodeQLCliServer, queryFile: stri
[`codeql/${language}-all`]: '*',
}
};
await fs.writeFile(path.join(queryPackDir, 'qlpack.yml'), yaml.safeDump(syntheticQueryPack));
await fs.writeFile(path.join(queryPackDir, 'qlpack.yml'), yaml.dump(syntheticQueryPack));
}
if (!language) {
throw new UserCancellationException('Could not determine language.');
@@ -139,7 +143,7 @@ async function findPackRoot(queryFile: string): Promise<string> {
while (!(await fs.pathExists(path.join(dir, 'qlpack.yml')))) {
dir = path.dirname(dir);
if (isFileSystemRoot(dir)) {
// there is no qlpack.yml in this direcory or any parent directory.
// there is no qlpack.yml in this directory or any parent directory.
// just use the query file's directory as the pack root.
return path.dirname(queryFile);
}
@@ -213,7 +217,7 @@ export async function runRemoteQuery(
if (!controllerRepo || !REPO_REGEX.test(controllerRepo)) {
void logger.log(controllerRepo ? 'Invalid controller repository name.' : 'No controller repository defined.');
controllerRepo = await window.showInputBox({
title: 'Controller repository in which to display progress and results of variant analysis',
title: 'Controller repository in which to run the GitHub Actions workflow for this variant analysis',
placeHolder: '<owner>/<repo>',
prompt: 'Enter the name of a GitHub repository in the format <owner>/<repo>',
ignoreFocusOut: true,
@@ -255,17 +259,19 @@ export async function runRemoteQuery(
});
const actionBranch = getActionBranch();
const workflowRunId = await runRemoteQueriesApiRequest(credentials, actionBranch, language, repoSelection, owner, repo, base64Pack, dryRun);
const apiResponse = await runRemoteQueriesApiRequest(credentials, actionBranch, language, repoSelection, owner, repo, base64Pack, dryRun);
const queryStartTime = Date.now();
const queryMetadata = await tryGetQueryMetadata(cliServer, queryFile);
if (dryRun) {
return { queryDirPath: remoteQueryDir.path };
} else {
if (!workflowRunId) {
if (!apiResponse) {
return;
}
const workflowRunId = apiResponse.workflow_run_id;
const repositoryCount = apiResponse.repositories_queried.length;
const remoteQuery = await buildRemoteQueryEntity(
queryFile,
queryMetadata,
@@ -273,7 +279,8 @@ export async function runRemoteQuery(
repo,
queryStartTime,
workflowRunId,
language);
language,
repositoryCount);
// don't return the path because it has been deleted
return { query: remoteQuery };
@@ -298,12 +305,13 @@ async function runRemoteQueriesApiRequest(
repo: string,
queryPackBase64: string,
dryRun = false
): Promise<void | number> {
): Promise<void | QueriesResponse> {
const data = {
ref,
language,
repositories: repoSelection.repositories ?? undefined,
repository_lists: repoSelection.repositoryLists ?? undefined,
repository_owners: repoSelection.owners ?? undefined,
query_pack: queryPackBase64,
};
@@ -332,33 +340,56 @@ async function runRemoteQueriesApiRequest(
);
const { popupMessage, logMessage } = parseResponse(owner, repo, response.data);
void showAndLogInformationMessage(popupMessage, { fullMessage: logMessage });
return response.data.workflow_run_id;
} catch (error) {
void showAndLogErrorMessage(getErrorMessage(error));
return response.data;
} catch (error: any) {
if (error.status === 404) {
void showAndLogErrorMessage(`Controller repository was not found. Please make sure it's a valid repo name.${eol}`);
} else {
void showAndLogErrorMessage(getErrorMessage(error));
}
}
}
const eol = os.EOL;
const eol2 = os.EOL + os.EOL;
// exported for testng only
// exported for testing only
export function parseResponse(owner: string, repo: string, response: QueriesResponse) {
const popupMessage = `Successfully scheduled runs. [Click here to see the progress](https://github.com/${owner}/${repo}/actions/runs/${response.workflow_run_id}).`
const repositoriesQueried = response.repositories_queried;
const repositoryCount = repositoriesQueried.length;
const popupMessage = `Successfully scheduled runs on ${pluralize(repositoryCount, 'repository', 'repositories')}. [Click here to see the progress](https://github.com/${owner}/${repo}/actions/runs/${response.workflow_run_id}).`
+ (response.errors ? `${eol2}Some repositories could not be scheduled. See extension log for details.` : '');
let logMessage = `Successfully scheduled runs. See https://github.com/${owner}/${repo}/actions/runs/${response.workflow_run_id}.`;
if (response.repositories_queried) {
logMessage += `${eol2}Repositories queried:${eol}${response.repositories_queried.join(', ')}`;
}
let logMessage = `Successfully scheduled runs on ${pluralize(repositoryCount, 'repository', 'repositories')}. See https://github.com/${owner}/${repo}/actions/runs/${response.workflow_run_id}.`;
logMessage += `${eol2}Repositories queried:${eol}${repositoriesQueried.join(', ')}`;
if (response.errors) {
const { invalid_repositories, repositories_without_database, private_repositories, cutoff_repositories, cutoff_repositories_count } = response.errors;
logMessage += `${eol2}Some repositories could not be scheduled.`;
if (response.errors.invalid_repositories?.length) {
logMessage += `${eol2}Invalid repositories:${eol}${response.errors.invalid_repositories.join(', ')}`;
if (invalid_repositories?.length) {
logMessage += `${eol2}${pluralize(invalid_repositories.length, 'repository', 'repositories')} invalid and could not be found:${eol}${invalid_repositories.join(', ')}`;
}
if (response.errors.repositories_without_database?.length) {
logMessage += `${eol2}Repositories without databases:${eol}${response.errors.repositories_without_database.join(', ')}`;
if (repositories_without_database?.length) {
logMessage += `${eol2}${pluralize(repositories_without_database.length, 'repository', 'repositories')} did not have a CodeQL database available:${eol}${repositories_without_database.join(', ')}`;
logMessage += `${eol}For each public repository that has not yet been added to the database service, we will try to create a database next time the store is updated.`;
}
if (private_repositories?.length) {
logMessage += `${eol2}${pluralize(private_repositories.length, 'repository', 'repositories')} not public:${eol}${private_repositories.join(', ')}`;
logMessage += `${eol}When using a public controller repository, only public repositories can be queried.`;
}
if (cutoff_repositories_count) {
logMessage += `${eol2}${pluralize(cutoff_repositories_count, 'repository', 'repositories')} over the limit for a single request`;
if (cutoff_repositories) {
logMessage += `:${eol}${cutoff_repositories.join(', ')}`;
if (cutoff_repositories_count !== cutoff_repositories.length) {
const moreRepositories = cutoff_repositories_count - cutoff_repositories.length;
logMessage += `${eol}...${eol}And another ${pluralize(moreRepositories, 'repository', 'repositories')}.`;
}
} else {
logMessage += '.';
}
logMessage += `${eol}Repositories were selected based on how recently they had been updated.`;
}
}
return {
@@ -378,7 +409,7 @@ export function parseResponse(owner: string, repo: string, response: QueriesResp
*/
async function ensureNameAndSuite(queryPackDir: string, packRelativePath: string): Promise<void> {
const packPath = path.join(queryPackDir, 'qlpack.yml');
const qlpack = yaml.safeLoad(await fs.readFile(packPath, 'utf8')) as QlPack;
const qlpack = yaml.load(await fs.readFile(packPath, 'utf8')) as QlPack;
delete qlpack.defaultSuiteFile;
qlpack.name = QUERY_PACK_NAME;
@@ -388,7 +419,7 @@ async function ensureNameAndSuite(queryPackDir: string, packRelativePath: string
}, {
query: packRelativePath.replace(/\\/g, '/')
}];
await fs.writeFile(packPath, yaml.safeDump(qlpack));
await fs.writeFile(packPath, yaml.dump(qlpack));
}
async function buildRemoteQueryEntity(
@@ -398,7 +429,8 @@ async function buildRemoteQueryEntity(
controllerRepoName: string,
queryStartTime: number,
workflowRunId: number,
language: string
language: string,
repositoryCount: number
): Promise<RemoteQuery> {
// The query name is either the name as specified in the query metadata, or the file name.
const queryName = queryMetadata?.name ?? path.basename(queryFilePath);
@@ -415,6 +447,7 @@ async function buildRemoteQueryEntity(
name: controllerRepoName,
},
executionStartTime: queryStartTime,
actionsWorkflowRunId: workflowRunId
actionsWorkflowRunId: workflowRunId,
repositoryCount,
};
}

View File

@@ -7,12 +7,16 @@ export interface AnalysisResults {
status: AnalysisResultStatus;
interpretedResults: AnalysisAlert[];
rawResults?: AnalysisRawResults;
resultCount: number,
starCount?: number,
lastUpdated?: number,
}
export interface AnalysisRawResults {
schema: ResultSetSchema;
resultSet: RawResultSet;
fileLinkPrefix: string;
sourceLocationPrefix: string;
capped: boolean;
}
@@ -86,3 +90,9 @@ export const getAnalysisResultCount = (analysisResults: AnalysisResults): number
const rawResultCount = analysisResults.rawResults?.resultSet.rows.length || 0;
return analysisResults.interpretedResults.length + rawResultCount;
};
/**
* Returns the total number of results for an analysis by adding all individual repo results.
*/
export const sumAnalysesResults = (analysesResults: AnalysisResults[]) =>
analysesResults.reduce((acc, curr) => acc + getAnalysisResultCount(curr), 0);

View File

@@ -2,25 +2,29 @@ import { DownloadLink } from '../download-link';
import { AnalysisFailure } from './analysis-failure';
export interface RemoteQueryResult {
queryTitle: string;
queryFileName: string;
queryFilePath: string;
queryText: string;
language: string;
workflowRunUrl: string;
totalRepositoryCount: number;
affectedRepositoryCount: number;
totalResultCount: number;
executionTimestamp: string;
executionDuration: string;
analysisSummaries: AnalysisSummary[];
analysisFailures: AnalysisFailure[];
queryId: string,
queryTitle: string,
queryFileName: string,
queryFilePath: string,
queryText: string,
language: string,
workflowRunUrl: string,
totalRepositoryCount: number,
affectedRepositoryCount: number,
totalResultCount: number,
executionTimestamp: string,
executionDuration: string,
analysisSummaries: AnalysisSummary[],
analysisFailures: AnalysisFailure[],
}
export interface AnalysisSummary {
nwo: string,
databaseSha: string,
resultCount: number,
sourceLocationPrefix: string,
downloadLink: DownloadLink,
fileSize: string,
starCount?: number,
lastUpdated?: number,
}

View File

@@ -1,13 +0,0 @@
module.exports = {
env: {
browser: true
},
extends: [
"plugin:react/recommended"
],
settings: {
react: {
version: 'detect'
}
}
}

View File

@@ -1,22 +0,0 @@
import * as React from 'react';
import styled from 'styled-components';
const Button = styled.button`
color: var(--vscode-button-foreground);
background-color: var(--vscode-button-background);
&:hover {
text-decoration: none;
background-color: var(--vscode-button-hoverBackground);
}
cursor: pointer;
padding: 8px;
border: 0;
`;
const ActionButton = ({ text, onClick }: { text: string, onClick: () => void }) => (
<Button onClick={onClick}>
{text}
</Button>
);
export default ActionButton;

View File

@@ -1,29 +0,0 @@
import * as React from 'react';
import styled from 'styled-components';
const BadgeContainer = styled.span`
justify-content: center;
align-items: center;
min-height: 100vh;
padding-left: 0.2em;
`;
const BadgeText = styled.span`
display: inline-block;
min-width: 1.5em;
padding: 0.3em;
border-radius: 35%;
font-size: x-small;
text-align: center;
background: var(--vscode-badge-background);
color: var(--vscode-badge-foreground);
border-color: var(--vscode-badge-background);
`;
const Badge = ({ text }: { text: string }) => (
<BadgeContainer>
<BadgeText>{text}</BadgeText>
</BadgeContainer>
);
export default Badge;

View File

@@ -1,228 +0,0 @@
import * as React from 'react';
import styled from 'styled-components';
import { CodeSnippet, FileLink, HighlightedRegion, AnalysisMessage, ResultSeverity } from '../shared/analysis-result';
import { Box, Link } from '@primer/react';
import VerticalSpace from './VerticalSpace';
import { createRemoteFileRef } from '../../pure/location-link-utils';
import { parseHighlightedLine, shouldHighlightLine } from '../../pure/sarif-utils';
const borderColor = 'var(--vscode-editor-snippetFinalTabstopHighlightBorder)';
const warningColor = '#966C23';
const highlightColor = 'var(--vscode-editor-findMatchHighlightBackground)';
const getSeverityColor = (severity: ResultSeverity) => {
switch (severity) {
case 'Recommendation':
return 'blue';
case 'Warning':
return warningColor;
case 'Error':
return 'red';
}
};
const replaceSpaceAndTabChar = (text: string) => text.replaceAll(' ', '\u00a0').replaceAll('\t', '\u00a0\u00a0\u00a0\u00a0');
const Container = styled.div`
font-family: ui-monospace, SFMono-Regular, SF Mono, Menlo, Consolas, Liberation Mono, monospace;
font-size: x-small;
`;
const TitleContainer = styled.div`
border: 0.1em solid ${borderColor};
border-top-left-radius: 0.2em;
border-top-right-radius: 0.2em;
padding: 0.5em;
`;
const CodeContainer = styled.div`
border-left: 0.1em solid ${borderColor};
border-right: 0.1em solid ${borderColor};
border-bottom: 0.1em solid ${borderColor};
border-bottom-left-radius: 0.2em;
border-bottom-right-radius: 0.2em;
padding-top: 1em;
padding-bottom: 1em;
`;
const MessageText = styled.div`
font-size: x-small;
padding-left: 0.5em;
`;
const MessageContainer = styled.div`
padding-top: 0.5em;
padding-bottom: 0.5em;
`;
const PlainLine = ({ text }: { text: string }) => {
return <span>{replaceSpaceAndTabChar(text)}</span>;
};
const HighlightedLine = ({ text }: { text: string }) => {
return <span style={{ backgroundColor: highlightColor }}>{replaceSpaceAndTabChar(text)}</span>;
};
const Message = ({
message,
currentLineNumber,
highlightedRegion,
borderColor,
children
}: {
message: AnalysisMessage,
currentLineNumber: number,
highlightedRegion?: HighlightedRegion,
borderColor: string,
children: React.ReactNode
}) => {
if (!highlightedRegion || highlightedRegion.endLine !== currentLineNumber) {
return <></>;
}
return <MessageContainer>
<Box
borderColor="border.default"
borderWidth={1}
borderStyle="solid"
borderLeftColor={borderColor}
borderLeftWidth={3}
paddingTop="1em"
paddingBottom="1em">
<MessageText>
{message.tokens.map((token, index) => {
switch (token.t) {
case 'text':
return <span key={`token-${index}`}>{token.text}</span>;
case 'location':
return <Link
key={`token-${index}`}
href={createRemoteFileRef(
token.location.fileLink,
token.location.highlightedRegion?.startLine,
token.location.highlightedRegion?.endLine)}>
{token.text}
</Link>;
default:
return <></>;
}
})}
{children && <>
<VerticalSpace size={2} />
{children}
</>
}
</MessageText>
</Box>
</MessageContainer>;
};
const CodeLine = ({
line,
lineNumber,
highlightedRegion
}: {
line: string,
lineNumber: number,
highlightedRegion?: HighlightedRegion
}) => {
if (!highlightedRegion || !shouldHighlightLine(lineNumber, highlightedRegion)) {
return <PlainLine text={line} />;
}
const partiallyHighlightedLine = parseHighlightedLine(line, lineNumber, highlightedRegion);
return (
<>
<PlainLine text={partiallyHighlightedLine.plainSection1} />
<HighlightedLine text={partiallyHighlightedLine.highlightedSection} />
<PlainLine text={partiallyHighlightedLine.plainSection2} />
</>
);
};
const FileCodeSnippet = ({
fileLink,
codeSnippet,
highlightedRegion,
severity,
message,
messageChildren,
}: {
fileLink: FileLink,
codeSnippet?: CodeSnippet,
highlightedRegion?: HighlightedRegion,
severity?: ResultSeverity,
message?: AnalysisMessage,
messageChildren?: React.ReactNode,
}) => {
const startingLine = codeSnippet?.startLine || 0;
const endingLine = codeSnippet?.endLine || 0;
const titleFileUri = createRemoteFileRef(
fileLink,
highlightedRegion?.startLine || startingLine,
highlightedRegion?.endLine || endingLine);
if (!codeSnippet) {
return (
<Container>
<TitleContainer>
<Link href={titleFileUri}>{fileLink.filePath}</Link>
</TitleContainer>
</Container>
);
}
const code = codeSnippet.text.split('\n');
return (
<Container>
<TitleContainer>
<Link href={titleFileUri}>{fileLink.filePath}</Link>
</TitleContainer>
<CodeContainer>
{code.map((line, index) => (
<div key={index}>
<Box display="flex">
<Box
p={2}
borderStyle="none"
paddingTop="0.01em"
paddingLeft="0.5em"
paddingRight="0.5em"
paddingBottom="0.2em">
{startingLine + index}
</Box>
<Box
flexGrow={1}
p={2}
borderStyle="none"
paddingTop="0.01em"
paddingLeft="1.5em"
paddingRight="0.5em"
paddingBottom="0.2em"
sx={{ wordBreak: 'break-word' }}>
<CodeLine
line={line}
lineNumber={startingLine + index}
highlightedRegion={highlightedRegion} />
</Box>
</Box>
{message && severity && <Message
message={message}
currentLineNumber={startingLine + index}
highlightedRegion={highlightedRegion}
borderColor={getSeverityColor(severity)}>
{messageChildren}
</Message>}
</div>
))}
</CodeContainer>
</Container>
);
};
export default FileCodeSnippet;

View File

@@ -1,30 +0,0 @@
import * as React from 'react';
import { ChangeEvent } from 'react';
import { TextInput } from '@primer/react';
import { SearchIcon } from '@primer/octicons-react';
interface RepositoriesSearchProps {
filterValue: string;
setFilterValue: (value: string) => void;
}
const RepositoriesSearch = ({ filterValue, setFilterValue }: RepositoriesSearchProps) => {
return <>
<TextInput
block
sx={{
backgroundColor: 'var(--vscode-editor-background);',
color: 'var(--vscode-editor-foreground);',
width: 'calc(100% - 14px)',
}}
leadingVisual={SearchIcon}
aria-label="Repository search"
name="repository-search"
placeholder="Filter by repository owner/name"
value={filterValue}
onChange={(e: ChangeEvent) => setFilterValue((e.target as HTMLInputElement).value)}
/>
</>;
};
export default RepositoriesSearch;

View File

@@ -1,18 +0,0 @@
{
"compilerOptions": {
"module": "esnext",
"moduleResolution": "node",
"target": "es6",
"outDir": "out",
"lib": ["ES2021", "dom"],
"jsx": "react",
"sourceMap": true,
"rootDir": "..",
"strict": true,
"noUnusedLocals": true,
"noImplicitReturns": true,
"noFallthroughCasesInSwitch": true,
"experimentalDecorators": true
},
"exclude": ["node_modules"]
}

View File

@@ -36,7 +36,8 @@ import { compileDatabaseUpgradeSequence, hasNondestructiveUpgradeCapabilities, u
import { ensureMetadataIsComplete } from './query-results';
import { SELECT_QUERY_NAME } from './contextual/locationFinder';
import { DecodedBqrsChunk } from './pure/bqrs-cli-types';
import { getErrorMessage } from './pure/helpers-pure';
import { asError, getErrorMessage } from './pure/helpers-pure';
import { generateSummarySymbolsFile } from './log-insights/summary-parser';
/**
* run-queries.ts
@@ -103,6 +104,14 @@ export class QueryEvaluationInfo {
return qsClient.findQueryEvalLogSummaryFile(this.querySaveDir);
}
get jsonEvalLogSummaryPath() {
return qsClient.findJsonQueryEvalLogSummaryFile(this.querySaveDir);
}
get evalLogSummarySymbolsPath() {
return qsClient.findQueryEvalLogSummarySymbolsFile(this.querySaveDir);
}
get evalLogEndSummaryPath() {
return qsClient.findQueryEvalLogEndSummaryFile(this.querySaveDir);
}
@@ -174,7 +183,7 @@ export class QueryEvaluationInfo {
db: dataset,
logPath: this.evalLogPath,
});
}
const params: messages.EvaluateQueriesParams = {
db: dataset,
@@ -199,21 +208,15 @@ export class QueryEvaluationInfo {
});
if (await this.hasEvalLog()) {
queryInfo.evalLogLocation = this.evalLogPath;
void qs.cliServer.generateLogSummary(this.evalLogPath, this.evalLogSummaryPath, this.evalLogEndSummaryPath)
.then(() => {
queryInfo.evalLogSummaryLocation = this.evalLogSummaryPath;
fs.readFile(this.evalLogEndSummaryPath, (err, buffer) => {
if (err) {
throw new Error(`Could not read structured evaluator log end of summary file at ${this.evalLogEndSummaryPath}.`);
}
void qs.logger.log(' --- Evaluator Log Summary --- ');
void qs.logger.log(buffer.toString());
});
})
queryInfo.evalLogSummaryLocation = await this.generateHumanReadableLogSummary(qs);
void this.logEndSummary(queryInfo.evalLogSummaryLocation, qs); // Logged asynchrnously
.catch(err => {
void showAndLogWarningMessage(`Failed to generate structured evaluator log summary. Reason: ${err.message}`);
});
if (config.isCanary()) { // Generate JSON summary for viewer.
await qs.cliServer.generateJsonLogSummary(this.evalLogPath, this.jsonEvalLogSummaryPath);
queryInfo.jsonEvalLogSummaryLocation = this.jsonEvalLogSummaryPath;
await generateSummarySymbolsFile(this.evalLogSummaryPath, this.evalLogSummarySymbolsPath);
queryInfo.evalLogSummarySymbolsLocation = this.evalLogSummarySymbolsPath;
}
} else {
void showAndLogWarningMessage(`Failed to write structured evaluator log to ${this.evalLogPath}.`);
}
@@ -248,7 +251,8 @@ export class QueryEvaluationInfo {
localChecking: false,
noComputeGetUrl: false,
noComputeToString: false,
computeDefaultStrings: true
computeDefaultStrings: true,
emitDebugInfo: true
},
extraOptions: {
timeoutSecs: qs.config.timeoutSecs
@@ -334,38 +338,111 @@ export class QueryEvaluationInfo {
/**
* Holds if this query already has a completed structured evaluator log
*/
async hasEvalLog(): Promise<boolean> {
async hasEvalLog(): Promise<boolean> {
return fs.pathExists(this.evalLogPath);
}
/**
* Calls the appropriate CLI command to generate a human-readable log summary.
* @param qs The query server client.
* @returns The path to the log summary, or `undefined` if the summary could not be generated.
*/
private async generateHumanReadableLogSummary(qs: qsClient.QueryServerClient): Promise<string | undefined> {
try {
await qs.cliServer.generateLogSummary(this.evalLogPath, this.evalLogSummaryPath, this.evalLogEndSummaryPath);
return this.evalLogSummaryPath;
} catch (e) {
const err = asError(e);
void showAndLogWarningMessage(`Failed to generate human-readable structured evaluator log summary. Reason: ${err.message}`);
return undefined;
}
}
/**
* Logs the end summary to the Output window and log file.
* @param logSummaryPath Path to the human-readable log summary
* @param qs The query server client.
*/
private async logEndSummary(logSummaryPath: string | undefined, qs: qsClient.QueryServerClient): Promise<void> {
if (logSummaryPath === undefined) {
// Failed to generate the log, so we don't expect an end summary either.
return;
}
try {
const endSummaryContent = await fs.readFile(this.evalLogEndSummaryPath, 'utf-8');
void qs.logger.log(' --- Evaluator Log Summary --- ', { additionalLogLocation: this.logPath });
void qs.logger.log(endSummaryContent, { additionalLogLocation: this.logPath });
} catch (e) {
void showAndLogWarningMessage(`Could not read structured evaluator log end of summary file at ${this.evalLogEndSummaryPath}.`);
}
}
/**
* Creates the CSV file containing the results of this query. This will only be called if the query
* does not have interpreted results and the CSV file does not already exist.
*
* @return Promise<true> if the operation creates the file. Promise<false> if the operation does
* not create the file.
*
* @throws Error if the operation fails.
*/
async exportCsvResults(qs: qsClient.QueryServerClient, csvPath: string, onFinish: () => void): Promise<void> {
async exportCsvResults(qs: qsClient.QueryServerClient, csvPath: string): Promise<boolean> {
const resultSet = await this.chooseResultSet(qs);
if (!resultSet) {
void showAndLogWarningMessage('Query has no result set.');
return false;
}
let stopDecoding = false;
const out = fs.createWriteStream(csvPath);
out.on('finish', onFinish);
out.on('error', () => {
if (!stopDecoding) {
stopDecoding = true;
void showAndLogErrorMessage(`Failed to write CSV results to ${csvPath}`);
}
const promise: Promise<boolean> = new Promise((resolve, reject) => {
out.on('finish', () => resolve(true));
out.on('error', () => {
if (!stopDecoding) {
stopDecoding = true;
reject(new Error(`Failed to write CSV results to ${csvPath}`));
}
});
});
let nextOffset: number | undefined = 0;
while (nextOffset !== undefined && !stopDecoding) {
const chunk: DecodedBqrsChunk = await qs.cliServer.bqrsDecode(this.resultsPaths.resultsPath, SELECT_QUERY_NAME, {
do {
const chunk: DecodedBqrsChunk = await qs.cliServer.bqrsDecode(this.resultsPaths.resultsPath, resultSet, {
pageSize: 100,
offset: nextOffset,
});
for (const tuple of chunk.tuples) {
out.write(tuple.join(',') + '\n');
}
chunk.tuples.forEach((tuple) => {
out.write(tuple.map((v, i) =>
chunk.columns[i].kind === 'String'
? `"${typeof v === 'string' ? v.replaceAll('"', '""') : v}"`
: v
).join(',') + '\n');
});
nextOffset = chunk.next;
}
} while (nextOffset && !stopDecoding);
out.end();
return promise;
}
/**
* Choose the name of the result set to run. If the `#select` set exists, use that. Otherwise,
* arbitrarily choose the first set. Most of the time, this will be correct.
*
* If the query has no result sets, then return undefined.
*/
async chooseResultSet(qs: qsClient.QueryServerClient) {
const resultSets = (await qs.cliServer.bqrsInfo(this.resultsPaths.resultsPath, 0))['result-sets'];
if (!resultSets.length) {
return undefined;
}
if (resultSets.find(r => r.name === SELECT_QUERY_NAME)) {
return SELECT_QUERY_NAME;
}
return resultSets[0].name;
}
/**
* Returns the path to the CSV alerts interpretation of this query results. If CSV results have
* not yet been produced, this will return first create the CSV results and then return the path.
@@ -753,10 +830,16 @@ export async function compileAndRunQueryAgainstDatabase(
const metadata = await tryGetQueryMetadata(cliServer, qlProgram.queryPath);
let availableMlModels: cli.MlModelInfo[] = [];
if (await cliServer.cliConstraints.supportsResolveMlModels()) {
if (!await cliServer.cliConstraints.supportsResolveMlModels()) {
void logger.log('Resolving ML models is unsupported by this version of the CLI. Running the query without any ML models.');
} else {
try {
availableMlModels = (await cliServer.resolveMlModels(diskWorkspaceFolders)).models;
void logger.log(`Found available ML models at the following paths: ${availableMlModels.map(x => `'${x.path}'`).join(', ')}.`);
availableMlModels = (await cliServer.resolveMlModels(diskWorkspaceFolders, initialInfo.queryPath)).models;
if (availableMlModels.length) {
void logger.log(`Found available ML models at the following paths: ${availableMlModels.map(x => `'${x.path}'`).join(', ')}.`);
} else {
void logger.log('Did not find any available ML models.');
}
} catch (e) {
const message = `Couldn't resolve available ML models for ${qlProgram.queryPath}. Running the ` +
`query without any ML models: ${e}.`;

View File

@@ -1,15 +1,16 @@
import * as React from 'react';
import { useState, useEffect } from 'react';
import * as Rdom from 'react-dom';
import {
ToCompareViewMessage,
SetComparisonsMessage,
} from '../../pure/interface-types';
import CompareSelector from './CompareSelector';
import { vscode } from '../../view/vscode-api';
import { vscode } from '../vscode-api';
import CompareTable from './CompareTable';
import '../results/resultsView.css';
const emptyComparison: SetComparisonsMessage = {
t: 'setComparisons',
stats: {},
@@ -75,10 +76,3 @@ export function Compare(_: Record<string, never>): JSX.Element {
return <div>Error!</div>;
}
}
Rdom.render(
<Compare />,
document.getElementById('root'),
// Post a message to the extension when fully loaded.
() => vscode.postMessage({ t: 'compareViewLoaded' })
);

View File

@@ -1,11 +1,11 @@
import * as React from 'react';
import { SetComparisonsMessage } from '../../pure/interface-types';
import RawTableHeader from '../../view/RawTableHeader';
import { className } from '../../view/result-table-utils';
import RawTableHeader from '../results/RawTableHeader';
import { className } from '../results/result-table-utils';
import { ResultRow } from '../../pure/bqrs-cli-types';
import RawTableRow from '../../view/RawTableRow';
import { vscode } from '../../view/vscode-api';
import RawTableRow from '../results/RawTableRow';
import { vscode } from '../vscode-api';
interface Props {
comparison: SetComparisonsMessage;

View File

@@ -0,0 +1,10 @@
import * as React from 'react';
import { WebviewDefinition } from '../webview-interface';
import { Compare } from './Compare';
const definition: WebviewDefinition = {
component: <Compare />,
loadedMessage: 'compareViewLoaded'
};
export default definition;

View File

@@ -1,5 +1,5 @@
import * as React from 'react';
import { AnalysisAlert } from '../shared/analysis-result';
import { AnalysisAlert } from '../../remote-queries/shared/analysis-result';
import CodePaths from './CodePaths';
import FileCodeSnippet from './FileCodeSnippet';

View File

@@ -1,9 +1,10 @@
import { TriangleDownIcon, XCircleIcon } from '@primer/octicons-react';
import { ActionList, ActionMenu, Box, Button, Label, Link, Overlay } from '@primer/react';
import { XCircleIcon } from '@primer/octicons-react';
import { Overlay } from '@primer/react';
import { VSCodeDropdown, VSCodeLink, VSCodeOption, VSCodeTag } from '@vscode/webview-ui-toolkit/react';
import * as React from 'react';
import { useRef, useState } from 'react';
import { ChangeEvent, useRef, useState } from 'react';
import styled from 'styled-components';
import { CodeFlow, AnalysisMessage, ResultSeverity } from '../shared/analysis-result';
import { CodeFlow, AnalysisMessage, ResultSeverity } from '../../remote-queries/shared/analysis-result';
import FileCodeSnippet from './FileCodeSnippet';
import SectionTitle from './SectionTitle';
import VerticalSpace from './VerticalSpace';
@@ -53,21 +54,21 @@ const CodePath = ({
<div key={`thread-flow-${index}`} style={{ maxWidth: '55em' }}>
{index !== 0 && <VerticalSpace size={3} />}
<Box display="flex" justifyContent="center" alignItems="center">
<Box flexGrow={1} p={0} border="none">
<div style={{ display: 'flex', justifyContent: 'center', alignItems: 'center' }}>
<div style={{ flexGrow: 1, padding: 0, border: 'none' }}>
<SectionTitle>Step {index + 1}</SectionTitle>
</Box>
</div>
{index === 0 &&
<Box p={0} border="none">
<Label>Source</Label>
</Box>
<div style={{ padding: 0, border: 'none' }}>
<VSCodeTag>Source</VSCodeTag>
</div>
}
{index === codeFlow.threadFlows.length - 1 &&
<Box p={0} border="none">
<Label>Sink</Label>
</Box>
<div style={{ padding: 0, border: 'none' }}>
<VSCodeTag>Sink</VSCodeTag>
</div>
}
</Box>
</div>
<VerticalSpace size={2} />
<FileCodeSnippet
@@ -93,25 +94,22 @@ const Menu = ({
codeFlows: CodeFlow[],
setSelectedCodeFlow: (value: React.SetStateAction<CodeFlow>) => void
}) => {
return <ActionMenu>
<ActionMenu.Anchor>
<Button variant="invisible" sx={{ fontWeight: 'normal', color: 'var(--vscode-editor-foreground);', padding: 0 }} >
{getCodeFlowName(codeFlows[0])}
<TriangleDownIcon size={16} />
</Button>
</ActionMenu.Anchor>
<ActionMenu.Overlay sx={{ backgroundColor: 'var(--vscode-editor-background)' }}>
<ActionList>
{codeFlows.map((codeFlow, index) =>
<ActionList.Item
key={`codeflow-${index}'`}
onSelect={(e: React.MouseEvent) => { setSelectedCodeFlow(codeFlow); }}>
{getCodeFlowName(codeFlow)}
</ActionList.Item>
)}
</ActionList>
</ActionMenu.Overlay>
</ActionMenu>;
return <VSCodeDropdown
onChange={(event: ChangeEvent<HTMLSelectElement>) => {
const selectedOption = event.target;
const selectedIndex = selectedOption.value as unknown as number;
setSelectedCodeFlow(codeFlows[selectedIndex]);
}}
>
{codeFlows.map((codeFlow, index) =>
<VSCodeOption
key={`codeflow-${index}'`}
value={index}
>
{getCodeFlowName(codeFlow)}
</VSCodeOption>
)}
</VSCodeDropdown>;
};
const CodePaths = ({
@@ -134,13 +132,13 @@ const CodePaths = ({
const closeOverlay = () => setIsOpen(false);
return (
<Box ref={anchorRef}>
<Link
<div ref={anchorRef}>
<VSCodeLink
onClick={() => setIsOpen(true)}
ref={linkRef}
sx={{ cursor: 'pointer' }}>
Show paths
</Link>
</VSCodeLink>
{isOpen && (
<Overlay
returnFocusRef={linkRef}
@@ -153,14 +151,14 @@ const CodePaths = ({
<SectionTitle>{ruleDescription}</SectionTitle>
<VerticalSpace size={2} />
<Box display="flex" justifyContent="center" alignItems="center">
<Box p={0} border="none">
<div style={{ display: 'flex', justifyContent: 'center', alignItems: 'center' }}>
<div style={{ padding: 0, border: 0 }}>
{codeFlows.length} paths available: {selectedCodeFlow.threadFlows.length} steps in
</Box>
<Box flexGrow={1} p={0} paddingLeft="0.2em" border="none">
</div>
<div style={{ flexGrow: 1, padding: 0, paddingLeft: '0.2em', border: 'none' }}>
<Menu codeFlows={codeFlows} setSelectedCodeFlow={setSelectedCodeFlow} />
</Box>
</Box>
</div>
</div>
<VerticalSpace size={2} />
<CodePath
@@ -173,7 +171,7 @@ const CodePaths = ({
</OverlayContainer>
</Overlay>
)}
</Box>
</div>
);
};

View File

@@ -1,19 +1,15 @@
import { Spinner } from '@primer/react';
import { VSCodeProgressRing } from '@vscode/webview-ui-toolkit/react';
import * as React from 'react';
import styled from 'styled-components';
const SpinnerContainer = styled.span`
vertical-align: middle;
svg {
width: 0.8em;
height: 0.8em;
}
display: inline-block;
`;
const DownloadSpinner = () => (
<SpinnerContainer>
<Spinner size="small" />
<VSCodeProgressRing style={{ height: '0.8em', width: '0.8em' }} />
</SpinnerContainer>
);

View File

@@ -0,0 +1,261 @@
import * as React from 'react';
import styled from 'styled-components';
import { CodeSnippet, FileLink, HighlightedRegion, AnalysisMessage, ResultSeverity } from '../../remote-queries/shared/analysis-result';
import VerticalSpace from './VerticalSpace';
import { createRemoteFileRef } from '../../pure/location-link-utils';
import { parseHighlightedLine, shouldHighlightLine } from '../../pure/sarif-utils';
import { VSCodeLink } from '@vscode/webview-ui-toolkit/react';
const borderColor = 'var(--vscode-editor-snippetFinalTabstopHighlightBorder)';
const warningColor = '#966C23';
const highlightColor = 'var(--vscode-editor-findMatchHighlightBackground)';
const getSeverityColor = (severity: ResultSeverity) => {
switch (severity) {
case 'Recommendation':
return 'blue';
case 'Warning':
return warningColor;
case 'Error':
return 'red';
}
};
const replaceSpaceAndTabChar = (text: string) => text.replaceAll(' ', '\u00a0').replaceAll('\t', '\u00a0\u00a0\u00a0\u00a0');
const Container = styled.div`
font-family: var(--vscode-editor-font-family);
font-size: small;
`;
const TitleContainer = styled.div`
border: 0.1em solid ${borderColor};
border-top-left-radius: 0.2em;
border-top-right-radius: 0.2em;
padding: 0.5em;
`;
const CodeContainer = styled.div`
border-left: 0.1em solid ${borderColor};
border-right: 0.1em solid ${borderColor};
border-bottom: 0.1em solid ${borderColor};
border-bottom-left-radius: 0.2em;
border-bottom-right-radius: 0.2em;
padding-top: 1em;
padding-bottom: 1em;
`;
const MessageText = styled.div`
font-size: small;
padding-left: 0.5em;
`;
const MessageContainer = styled.div`
padding-top: 0.5em;
padding-bottom: 0.5em;
`;
const PlainCode = ({ text }: { text: string }) => {
return <span>{replaceSpaceAndTabChar(text)}</span>;
};
const HighlightedCode = ({ text }: { text: string }) => {
return <span style={{ backgroundColor: highlightColor }}>{replaceSpaceAndTabChar(text)}</span>;
};
const Message = ({
message,
borderLeftColor,
children
}: {
message: AnalysisMessage,
borderLeftColor: string,
children: React.ReactNode
}) => {
return <div style={{
borderColor: borderColor,
borderWidth: '0.1em',
borderStyle: 'solid',
borderLeftColor: borderLeftColor,
borderLeftWidth: '0.3em',
paddingTop: '1em',
paddingBottom: '1em'
}}>
<MessageText>
{message.tokens.map((token, index) => {
switch (token.t) {
case 'text':
return <span key={`token-${index}`}>{token.text}</span>;
case 'location':
return <VSCodeLink
style={{ fontFamily: 'var(--vscode-editor-font-family)' }}
key={`token-${index}`}
href={createRemoteFileRef(
token.location.fileLink,
token.location.highlightedRegion?.startLine,
token.location.highlightedRegion?.endLine)}>
{token.text}
</VSCodeLink>;
default:
return <></>;
}
})}
{children && <>
<VerticalSpace size={2} />
{children}
</>
}
</MessageText>
</div>;
};
const Code = ({
line,
lineNumber,
highlightedRegion
}: {
line: string,
lineNumber: number,
highlightedRegion?: HighlightedRegion
}) => {
if (!highlightedRegion || !shouldHighlightLine(lineNumber, highlightedRegion)) {
return <PlainCode text={line} />;
}
const partiallyHighlightedLine = parseHighlightedLine(line, lineNumber, highlightedRegion);
return (
<>
<PlainCode text={partiallyHighlightedLine.plainSection1} />
<HighlightedCode text={partiallyHighlightedLine.highlightedSection} />
<PlainCode text={partiallyHighlightedLine.plainSection2} />
</>
);
};
const Line = ({
line,
lineIndex,
startingLineIndex,
highlightedRegion,
severity,
message,
messageChildren
}: {
line: string,
lineIndex: number,
startingLineIndex: number,
highlightedRegion?: HighlightedRegion,
severity?: ResultSeverity,
message?: AnalysisMessage,
messageChildren?: React.ReactNode,
}) => {
const shouldShowMessage = message &&
severity &&
highlightedRegion &&
highlightedRegion.endLine == startingLineIndex + lineIndex;
return <div>
<div style={{ display: 'flex' }} >
<div style={{
borderStyle: 'none',
paddingTop: '0.01em',
paddingLeft: '0.5em',
paddingRight: '0.5em',
paddingBottom: '0.2em'
}}>
{startingLineIndex + lineIndex}
</div>
<div style={{
flexGrow: 1,
borderStyle: 'none',
paddingTop: '0.01em',
paddingLeft: '1.5em',
paddingRight: '0.5em',
paddingBottom: '0.2em',
wordBreak: 'break-word'
}}>
<Code
line={line}
lineNumber={startingLineIndex + lineIndex}
highlightedRegion={highlightedRegion} />
</div>
</div>
{shouldShowMessage &&
<MessageContainer>
<Message
message={message}
borderLeftColor={getSeverityColor(severity)}>
{messageChildren}
</Message>
</MessageContainer>
}
</div>;
};
const FileCodeSnippet = ({
fileLink,
codeSnippet,
highlightedRegion,
severity,
message,
messageChildren,
}: {
fileLink: FileLink,
codeSnippet?: CodeSnippet,
highlightedRegion?: HighlightedRegion,
severity?: ResultSeverity,
message?: AnalysisMessage,
messageChildren?: React.ReactNode,
}) => {
const startingLine = codeSnippet?.startLine || 0;
const endingLine = codeSnippet?.endLine || 0;
const titleFileUri = createRemoteFileRef(
fileLink,
highlightedRegion?.startLine || startingLine,
highlightedRegion?.endLine || endingLine);
if (!codeSnippet) {
return (
<Container>
<TitleContainer>
<VSCodeLink href={titleFileUri}>{fileLink.filePath}</VSCodeLink>
</TitleContainer>
{message && severity &&
<Message
message={message}
borderLeftColor={getSeverityColor(severity)}>
{messageChildren}
</Message>}
</Container>
);
}
const code = codeSnippet.text.split('\n');
return (
<Container>
<TitleContainer>
<VSCodeLink href={titleFileUri}>{fileLink.filePath}</VSCodeLink>
</TitleContainer>
<CodeContainer>
{code.map((line, index) => (
<Line
key={`line-${index}`}
line={line}
lineIndex={index}
startingLineIndex={startingLine}
highlightedRegion={highlightedRegion}
severity={severity}
message={message}
messageChildren={messageChildren}
/>
))}
</CodeContainer>
</Container>
);
};
export default FileCodeSnippet;

View File

@@ -0,0 +1,38 @@
import * as React from 'react';
import { RepoPushIcon } from '@primer/octicons-react';
import styled from 'styled-components';
import { humanizeRelativeTime } from '../../pure/time';
const IconContainer = styled.span`
flex-grow: 0;
text-align: right;
margin-right: 0;
`;
const Duration = styled.span`
text-align: left;
width: 8em;
margin-left: 0.5em;
`;
type Props = { lastUpdated?: number };
const LastUpdated = ({ lastUpdated }: Props) => (
// lastUpdated will be undefined for older results that were
// created before the lastUpdated field was added.
Number.isFinite(lastUpdated) ? (
<>
<IconContainer>
<RepoPushIcon size={16} />
</IconContainer>
<Duration>
{humanizeRelativeTime(lastUpdated)}
</Duration>
</>
) : (
<></>
)
);
export default LastUpdated;

View File

@@ -1,41 +1,48 @@
import * as React from 'react';
import { Box, Link } from '@primer/react';
import { VSCodeLink } from '@vscode/webview-ui-toolkit/react';
import { CellValue, RawResultSet, ResultSetSchema } from '../../pure/bqrs-cli-types';
import { tryGetRemoteLocation } from '../../pure/bqrs-utils';
import { useState } from 'react';
import TextButton from './TextButton';
import { convertNonPrintableChars } from '../../text-utils';
const borderColor = 'var(--vscode-editor-snippetFinalTabstopHighlightBorder)';
const numOfResultsInContractedMode = 5;
const Row = ({
row,
fileLinkPrefix
fileLinkPrefix,
sourceLocationPrefix
}: {
row: CellValue[],
fileLinkPrefix: string
fileLinkPrefix: string,
sourceLocationPrefix: string
}) => (
<>
{row.map((cell, cellIndex) => (
<Box key={cellIndex}
borderColor="border.default"
borderStyle="solid"
justifyContent="center"
alignItems="center"
p={2}
sx={{ wordBreak: 'break-word' }}>
<Cell value={cell} fileLinkPrefix={fileLinkPrefix} />
</Box>
<div key={cellIndex} style={{
borderColor: borderColor,
borderStyle: 'solid',
justifyContent: 'center',
alignItems: 'center',
padding: '0.4rem',
wordBreak: 'break-word'
}}>
<Cell value={cell} fileLinkPrefix={fileLinkPrefix} sourceLocationPrefix={sourceLocationPrefix} />
</div>
))}
</>
);
const Cell = ({
value,
fileLinkPrefix
fileLinkPrefix,
sourceLocationPrefix
}: {
value: CellValue,
fileLinkPrefix: string
sourceLocationPrefix: string
}) => {
switch (typeof value) {
case 'string':
@@ -43,8 +50,13 @@ const Cell = ({
case 'boolean':
return <span>{convertNonPrintableChars(value.toString())}</span>;
case 'object': {
const url = tryGetRemoteLocation(value.url, fileLinkPrefix);
return <Link href={url}>{convertNonPrintableChars(value.label)}</Link>;
const url = tryGetRemoteLocation(value.url, fileLinkPrefix, sourceLocationPrefix);
const safeLabel = convertNonPrintableChars(value.label);
if (url) {
return <VSCodeLink href={url}>{safeLabel}</VSCodeLink>;
} else {
return <span>{safeLabel}</span>;
}
}
}
};
@@ -52,11 +64,13 @@ const Cell = ({
const RawResultsTable = ({
schema,
results,
fileLinkPrefix
fileLinkPrefix,
sourceLocationPrefix
}: {
schema: ResultSetSchema,
results: RawResultSet,
fileLinkPrefix: string
fileLinkPrefix: string,
sourceLocationPrefix: string
}) => {
const [tableExpanded, setTableExpanded] = useState(false);
const numOfResultsToShow = tableExpanded ? results.rows.length : numOfResultsInContractedMode;
@@ -69,15 +83,16 @@ const RawResultsTable = ({
return (
<>
<Box
display="grid"
gridTemplateColumns={gridTemplateColumns}
maxWidth="45rem"
p={2}>
<div style={{
display: 'grid',
gridTemplateColumns: gridTemplateColumns,
maxWidth: '45rem',
padding: '0.4rem'
}}>
{results.rows.slice(0, numOfResultsToShow).map((row, rowIndex) => (
<Row key={rowIndex} row={row} fileLinkPrefix={fileLinkPrefix} />
<Row key={rowIndex} row={row} fileLinkPrefix={fileLinkPrefix} sourceLocationPrefix={sourceLocationPrefix} />
))}
</Box>
</div>
{
showButton &&
<TextButton size='x-small' onClick={() => setTableExpanded(!tableExpanded)}>

View File

@@ -1,30 +1,35 @@
import * as React from 'react';
import { useEffect, useState } from 'react';
import * as Rdom from 'react-dom';
import { Box, Flash, ThemeProvider } from '@primer/react';
import { Flash, ThemeProvider } from '@primer/react';
import { ToRemoteQueriesMessage } from '../../pure/interface-types';
import { AnalysisSummary, RemoteQueryResult } from '../shared/remote-query-result';
import { MAX_RAW_RESULTS } from '../shared/result-limits';
import { vscode } from '../../view/vscode-api';
import { AnalysisSummary, RemoteQueryResult } from '../../remote-queries/shared/remote-query-result';
import { MAX_RAW_RESULTS } from '../../remote-queries/shared/result-limits';
import { vscode } from '../vscode-api';
import { VSCodeBadge, VSCodeButton } from '@vscode/webview-ui-toolkit/react';
import SectionTitle from './SectionTitle';
import VerticalSpace from './VerticalSpace';
import HorizontalSpace from './HorizontalSpace';
import Badge from './Badge';
import ViewTitle from './ViewTitle';
import DownloadButton from './DownloadButton';
import { AnalysisResults, getAnalysisResultCount } from '../shared/analysis-result';
import { AnalysisResults, getAnalysisResultCount } from '../../remote-queries/shared/analysis-result';
import DownloadSpinner from './DownloadSpinner';
import CollapsibleItem from './CollapsibleItem';
import { AlertIcon, CodeSquareIcon, FileCodeIcon, RepoIcon, TerminalIcon } from '@primer/octicons-react';
import AnalysisAlertResult from './AnalysisAlertResult';
import RawResultsTable from './RawResultsTable';
import RepositoriesSearch from './RepositoriesSearch';
import ActionButton from './ActionButton';
import StarCount from './StarCount';
import SortRepoFilter, { Sort, sorter } from './SortRepoFilter';
import LastUpdated from './LastUpdated';
import RepoListCopyButton from './RepoListCopyButton';
import './baseStyles.css';
import './remoteQueries.css';
const numOfReposInContractedMode = 10;
const emptyQueryResult: RemoteQueryResult = {
queryId: '',
queryTitle: '',
queryFileName: '',
queryFilePath: '',
@@ -68,14 +73,18 @@ const openQueryTextVirtualFile = (queryResult: RemoteQueryResult) => {
});
};
function createResultsDescription(queryResult: RemoteQueryResult) {
const reposCount = `${queryResult.totalRepositoryCount} ${queryResult.totalRepositoryCount === 1 ? 'repository' : 'repositories'}`;
return `${queryResult.totalResultCount} results from running against ${reposCount} (${queryResult.executionDuration}), ${queryResult.executionTimestamp}`;
}
const sumAnalysesResults = (analysesResults: AnalysisResults[]) =>
analysesResults.reduce((acc, curr) => acc + getAnalysisResultCount(curr), 0);
const QueryInfo = (queryResult: RemoteQueryResult) => (
<>
<VerticalSpace size={1} />
{queryResult.totalResultCount} results from running against {queryResult.totalRepositoryCount} repositories
({queryResult.executionDuration}), {queryResult.executionTimestamp}
{createResultsDescription(queryResult)}
<VerticalSpace size={1} />
<span>
<a className="vscode-codeql__query-info-link" href="#" onClick={() => openQueryFile(queryResult)}>
@@ -125,10 +134,14 @@ const Failures = (queryResult: RemoteQueryResult) => {
const SummaryTitleWithResults = ({
queryResult,
analysesResults
analysesResults,
sort,
setSort
}: {
queryResult: RemoteQueryResult,
analysesResults: AnalysisResults[]
analysesResults: AnalysisResults[],
sort: Sort,
setSort: (sort: Sort) => void
}) => {
const showDownloadButton = queryResult.totalResultCount !== sumAnalysesResults(analysesResults);
@@ -140,6 +153,14 @@ const SummaryTitleWithResults = ({
text="Download all"
onClick={() => downloadAllAnalysesResults(queryResult)} />
}
<div style={{ flexGrow: 2, textAlign: 'right' }}>
<RepoListCopyButton queryResult={queryResult} />
<HorizontalSpace size={1} />
<SortRepoFilter
sort={sort}
setSort={setSort}
/>
</div>
</div>
);
};
@@ -180,24 +201,33 @@ const SummaryItem = ({
analysisSummary: AnalysisSummary,
analysisResults: AnalysisResults | undefined
}) => (
<span>
<>
<span className="vscode-codeql__analysis-item"><RepoIcon size={16} /></span>
<span className="vscode-codeql__analysis-item">{analysisSummary.nwo}</span>
<span className="vscode-codeql__analysis-item"><Badge text={analysisSummary.resultCount.toString()} /></span>
<HorizontalSpace size={1} />
<span className="vscode-codeql__analysis-item">
<VSCodeBadge>{analysisSummary.resultCount.toString()}</VSCodeBadge>
</span>
<span className="vscode-codeql__analysis-item">
<SummaryItemDownload
analysisSummary={analysisSummary}
analysisResults={analysisResults} />
</span>
</span>
<StarCount starCount={analysisSummary.starCount} />
<LastUpdated lastUpdated={analysisSummary.lastUpdated} />
</>
);
const Summary = ({
queryResult,
analysesResults
analysesResults,
sort,
setSort
}: {
queryResult: RemoteQueryResult,
analysesResults: AnalysisResults[]
analysesResults: AnalysisResults[],
sort: Sort,
setSort: (sort: Sort) => void
}) => {
const [repoListExpanded, setRepoListExpanded] = useState(false);
const numOfReposToShow = repoListExpanded ? queryResult.analysisSummaries.length : numOfReposInContractedMode;
@@ -209,17 +239,21 @@ const Summary = ({
? <SummaryTitleNoResults />
: <SummaryTitleWithResults
queryResult={queryResult}
analysesResults={analysesResults} />
analysesResults={analysesResults}
sort={sort}
setSort={setSort} />
}
<ul className="vscode-codeql__flat-list">
{queryResult.analysisSummaries.slice(0, numOfReposToShow).map((summary, i) =>
<li key={summary.nwo} className="vscode-codeql__analysis-summaries-list-item">
<SummaryItem
analysisSummary={summary}
analysisResults={analysesResults.find(a => a.nwo === summary.nwo)} />
</li>
)}
{queryResult.analysisSummaries.slice(0, numOfReposToShow)
.sort(sorter(sort))
.map((summary, i) =>
<li key={summary.nwo} className="vscode-codeql__analysis-summaries-list-item">
<SummaryItem
analysisSummary={summary}
analysisResults={analysesResults.find(a => a.nwo === summary.nwo)} />
</li>
)}
</ul>
{
queryResult.analysisSummaries.length > numOfReposInContractedMode &&
@@ -279,7 +313,8 @@ const RepoAnalysisResults = (analysisResults: AnalysisResults) => {
const numOfResults = getAnalysisResultCount(analysisResults);
const title = <>
{analysisResults.nwo}
<Badge text={numOfResults.toString()} />
<HorizontalSpace size={1} />
<VSCodeBadge>{numOfResults.toString()}</VSCodeBadge>
</>;
return (
@@ -295,7 +330,8 @@ const RepoAnalysisResults = (analysisResults: AnalysisResults) => {
<RawResultsTable
schema={analysisResults.rawResults.schema}
results={analysisResults.rawResults.resultSet}
fileLinkPrefix={analysisResults.rawResults.fileLinkPrefix} />
fileLinkPrefix={analysisResults.rawResults.fileLinkPrefix}
sourceLocationPrefix={analysisResults.rawResults.sourceLocationPrefix} />
}
</CollapsibleItem>
);
@@ -304,11 +340,13 @@ const RepoAnalysisResults = (analysisResults: AnalysisResults) => {
const AnalysesResults = ({
queryResult,
analysesResults,
totalResults
totalResults,
sort,
}: {
queryResult: RemoteQueryResult,
analysesResults: AnalysisResults[],
totalResults: number
totalResults: number,
sort: Sort
}) => {
const totalAnalysesResults = sumAnalysesResults(analysesResults);
const [filterValue, setFilterValue] = React.useState('');
@@ -320,16 +358,16 @@ const AnalysesResults = ({
return (
<>
<VerticalSpace size={2} />
<Box display="flex">
<Box flexGrow={1}>
<div style={{ display: 'flex' }}>
<div style={{ flexGrow: 1 }}>
<AnalysesResultsTitle
totalAnalysesResults={totalAnalysesResults}
totalResults={totalResults} />
</Box>
<Box>
<ActionButton text="Export all" onClick={exportResults}></ActionButton>
</Box>
</Box>
</div>
<div>
<VSCodeButton onClick={exportResults}>Export all</VSCodeButton>
</div>
</div>
<AnalysesResultsDescription
queryResult={queryResult}
analysesResults={analysesResults} />
@@ -343,6 +381,7 @@ const AnalysesResults = ({
{analysesResults
.filter(a => a.interpretedResults.length > 0 || a.rawResults)
.filter(a => a.nwo.toLowerCase().includes(filterValue.toLowerCase()))
.sort(sorter(sort))
.map(r =>
<li key={r.nwo} className="vscode-codeql__analyses-results-list-item">
<RepoAnalysisResults {...r} />
@@ -355,6 +394,7 @@ const AnalysesResults = ({
export function RemoteQueries(): JSX.Element {
const [queryResult, setQueryResult] = useState<RemoteQueryResult>(emptyQueryResult);
const [analysesResults, setAnalysesResults] = useState<AnalysisResults[]>([]);
const [sort, setSort] = useState<Sort>('name');
useEffect(() => {
window.addEventListener('message', (evt: MessageEvent) => {
@@ -384,11 +424,16 @@ export function RemoteQueries(): JSX.Element {
<ViewTitle>{queryResult.queryTitle}</ViewTitle>
<QueryInfo {...queryResult} />
<Failures {...queryResult} />
<Summary queryResult={queryResult} analysesResults={analysesResults} />
<Summary
queryResult={queryResult}
analysesResults={analysesResults}
sort={sort}
setSort={setSort} />
<AnalysesResults
queryResult={queryResult}
analysesResults={analysesResults}
totalResults={queryResult.totalResultCount} />
totalResults={queryResult.totalResultCount}
sort={sort} />
</ThemeProvider>
</div>
);
@@ -397,10 +442,3 @@ export function RemoteQueries(): JSX.Element {
return <div>There was an error displaying the view.</div>;
}
}
Rdom.render(
<RemoteQueries />,
document.getElementById('root'),
// Post a message to the extension when fully loaded.
() => vscode.postMessage({ t: 'remoteQueryLoaded' })
);

View File

@@ -0,0 +1,24 @@
import * as React from 'react';
import { vscode } from '../vscode-api';
import { RemoteQueryResult } from '../../remote-queries/shared/remote-query-result';
import { CopyIcon } from '@primer/octicons-react';
import { IconButton } from '@primer/react';
const copyRepositoryList = (queryResult: RemoteQueryResult) => {
vscode.postMessage({
t: 'copyRepoList',
queryId: queryResult.queryId
});
};
const RepoListCopyButton = ({ queryResult }: { queryResult: RemoteQueryResult }) => (
<IconButton
aria-label="Copy repository list"
icon={CopyIcon}
variant="invisible"
size="small"
sx={{ 'text-align': 'right' }}
onClick={() => copyRepositoryList(queryResult)} />
);
export default RepoListCopyButton;

View File

@@ -0,0 +1,24 @@
import * as React from 'react';
import { VSCodeTextField } from '@vscode/webview-ui-toolkit/react';
interface RepositoriesSearchProps {
filterValue: string;
setFilterValue: (value: string) => void;
}
const RepositoriesSearch = ({ filterValue, setFilterValue }: RepositoriesSearchProps) => {
return <>
<VSCodeTextField
style={{ width: '100%' }}
placeholder='Filter by repository owner/name'
ariaLabel="Repository search"
name="repository-search"
value={filterValue}
onInput={(e: InputEvent) => setFilterValue((e.target as HTMLInputElement).value)}
>
<span slot="start" className="codicon codicon-search"></span>
</VSCodeTextField>
</>;
};
export default RepositoriesSearch;

View File

@@ -0,0 +1,83 @@
import * as React from 'react';
import { FilterIcon } from '@primer/octicons-react';
import { ActionList, ActionMenu, IconButton } from '@primer/react';
import styled from 'styled-components';
const SortWrapper = styled.span`
flex-grow: 2;
text-align: right;
margin-right: 0;
`;
export type Sort = 'name' | 'stars' | 'results' | 'lastUpdated';
type Props = {
sort: Sort;
setSort: (sort: Sort) => void;
};
type Sortable = {
nwo: string;
starCount?: number;
resultCount?: number;
lastUpdated?: number;
};
const sortBy = [
{ name: 'Sort by Name', sort: 'name' },
{ name: 'Sort by Results', sort: 'results' },
{ name: 'Sort by Stars', sort: 'stars' },
{ name: 'Sort by Last Updated', sort: 'lastUpdated' },
];
export function sorter(sort: Sort): (left: Sortable, right: Sortable) => number {
// stars and results are highest to lowest
// name is alphabetical
return (left: Sortable, right: Sortable) => {
if (sort === 'stars') {
const stars = (right.starCount || 0) - (left.starCount || 0);
if (stars !== 0) {
return stars;
}
}
if (sort === 'lastUpdated') {
const lastUpdated = (right.lastUpdated || 0) - (left.lastUpdated || 0);
if (lastUpdated !== 0) {
return lastUpdated;
}
}
if (sort === 'results') {
const results = (right.resultCount || 0) - (left.resultCount || 0);
if (results !== 0) {
return results;
}
}
// Fall back on name compare if results, stars, or lastUpdated are equal
return left.nwo.localeCompare(right.nwo, undefined, { sensitivity: 'base' });
};
}
const SortRepoFilter = ({ sort, setSort }: Props) => {
return <SortWrapper>
<ActionMenu>
<ActionMenu.Anchor>
<IconButton icon={FilterIcon} variant="invisible" aria-label="Sort results" />
</ActionMenu.Anchor>
<ActionMenu.Overlay width="small" anchorSide="outside-bottom">
<ActionList selectionVariant="single">
{sortBy.map((type, index) => (
<ActionList.Item
key={index}
selected={type.sort === sort} onSelect={() => setSort(type.sort as Sort)}
>
{type.name}
</ActionList.Item>
))}
</ActionList>
</ActionMenu.Overlay>
</ActionMenu>
</SortWrapper>;
};
export default SortRepoFilter;

View File

@@ -0,0 +1,45 @@
import * as React from 'react';
import { StarIcon } from '@primer/octicons-react';
import styled from 'styled-components';
const Star = styled.span`
flex-grow: 2;
text-align: right;
margin-right: 0;
`;
const Count = styled.span`
text-align: left;
width: 2em;
margin-left: 0.5em;
margin-right: 1.5em;
`;
type Props = { starCount?: number };
const StarCount = ({ starCount }: Props) => (
Number.isFinite(starCount) ? (
<>
<Star>
<StarIcon size={16} />
</Star>
<Count>
{displayStars(starCount!)}
</Count>
</>
) : (
<></>
)
);
function displayStars(starCount: number) {
if (starCount > 10000) {
return `${(starCount / 1000).toFixed(0)}k`;
}
if (starCount > 1000) {
return `${(starCount / 1000).toFixed(1)}k`;
}
return starCount.toFixed(0);
}
export default StarCount;

View File

@@ -0,0 +1,10 @@
import * as React from 'react';
import { WebviewDefinition } from '../webview-interface';
import { RemoteQueries } from './RemoteQueries';
const definition: WebviewDefinition = {
component: <RemoteQueries />,
loadedMessage: 'remoteQueryLoaded'
};
export default definition;

View File

@@ -14,10 +14,12 @@
.vscode-codeql__query-summary-container {
padding-top: 1.5em;
display: flex;
}
.vscode-codeql__analysis-summaries-list-item {
margin-top: 0.5em;
display: flex;
}
.vscode-codeql__analyses-results-list-item {

View File

@@ -1,9 +1,9 @@
import * as React from 'react';
import { vscode } from './vscode-api';
import { RawResultsSortState, SortDirection } from '../pure/interface-types';
import { vscode } from '../vscode-api';
import { RawResultsSortState, SortDirection } from '../../pure/interface-types';
import { nextSortDirection } from './result-table-utils';
import { Column } from '../pure/bqrs-cli-types';
import { Column } from '../../pure/bqrs-cli-types';
interface Props {
readonly columns: readonly Column[];

View File

@@ -1,5 +1,5 @@
import * as React from 'react';
import { ResultRow } from '../pure/bqrs-cli-types';
import { ResultRow } from '../../pure/bqrs-cli-types';
import { zebraStripe } from './result-table-utils';
import RawTableValue from './RawTableValue';

Some files were not shown because too many files have changed in this diff Show More