Unpack the request/response payload contents in the shell and keep annotated logs
This commit is contained in:
Michael Hohn
committed by
=Michael Hohn
=Michael Hohn
parent
6b7a21d4d1
commit
5f235371f5
29
README.org
29
README.org
@@ -193,26 +193,35 @@
|
||||
|
||||
./gh-mrva -h
|
||||
|
||||
# Submit the mrva job
|
||||
SN=10
|
||||
# In log-submit-the-mrva-job.log after edit
|
||||
SN=11
|
||||
./gh-mrva submit --language cpp --session mirva-session-$SN \
|
||||
--list mirva-list \
|
||||
--query /Users/hohn/local/gh-mrva/FlatBuffersFunc.ql >& log-$SN.out &
|
||||
tail -f log-$SN.out
|
||||
|
||||
|
||||
# Check the status
|
||||
./gh-mrva status --session mirva-session-$SN |& tee log-$SN-status.out
|
||||
|
||||
# Download the sarif files when finished
|
||||
gh mrva download --session mirva-session-$SN \
|
||||
--output-dir mirva-session-$SN-sarif
|
||||
|
||||
# Or download the sarif files and CodeQL dbs when finished
|
||||
gh mrva download --session mirva-session-$SN \
|
||||
# Download the sarif files and CodeQL dbs when finished
|
||||
./gh-mrva download --session mirva-session-$SN \
|
||||
--download-dbs \
|
||||
--output-dir mirva-session-$SN-sarif
|
||||
--output-dir mirva-session-$SN-sarif \
|
||||
|& tee log-download.log
|
||||
# 2024/02/08 15:33:39 >> Response body is
|
||||
# Zip archive data, at least v1.0 to extract, compression method=deflate
|
||||
# 0:$ unzip -v foo
|
||||
# Archive: foo
|
||||
# Length Method Size Cmpr Date Time CRC-32 Name
|
||||
# -------- ------ ------- ---- ---------- ----- -------- ----
|
||||
# 2297 Defl:N 980 57% 02-08-2024 22:54 9465f5ff results.sarif
|
||||
# 148 Defl:N 121 18% 02-08-2024 22:54 dc8df445 results.bqrs
|
||||
# -------- ------- --- -------
|
||||
# 2445 1101 55% 2 files
|
||||
|
||||
# 0 4 Local file header signature = 0x04034b50 (PK♥♦ or "PK\3\4") PK
|
||||
# 0x50, 0x4b, 0x03, 0x04
|
||||
# byteArray := []byte{ 0x50, 0x4b, 0x03, 0x04 }
|
||||
#+END_SRC
|
||||
|
||||
** Use the delve debugger to find sigsev
|
||||
|
||||
4161
log-download.log
Normal file
4161
log-download.log
Normal file
File diff suppressed because it is too large
Load Diff
151
log-status.log
Normal file
151
log-status.log
Normal file
@@ -0,0 +1,151 @@
|
||||
# Check the status
|
||||
./gh-mrva status --session mirva-session-$SN |& tee log-$SN-status.out
|
||||
|
||||
|
||||
2024/02/08 15:27:29 >> GET https://api.github.com/repos/hohn/mirva-controller/code-scanning/codeql/variant-analyses/8985
|
||||
2024/02/08 15:27:29 << 200 https://api.github.com/repos/hohn/mirva-controller/code-scanning/codeql/variant-analyses/8985
|
||||
2024/02/08 15:27:29 >> Response body: {
|
||||
"id": 8985,
|
||||
"controller_repo": {
|
||||
"id": 747492529,
|
||||
"node_id": "R_kgDOLI3UsQ",
|
||||
"name": "mirva-controller",
|
||||
"full_name": "hohn/mirva-controller",
|
||||
"private": false,
|
||||
"owner": {
|
||||
"login": "hohn",
|
||||
"id": 2253228,
|
||||
"node_id": "MDQ6VXNlcjIyNTMyMjg=",
|
||||
"avatar_url": "https://avatars.githubusercontent.com/u/2253228?v=4",
|
||||
"gravatar_id": "",
|
||||
"url": "https://api.github.com/users/hohn",
|
||||
"html_url": "https://github.com/hohn",
|
||||
"followers_url": "https://api.github.com/users/hohn/followers",
|
||||
"following_url": "https://api.github.com/users/hohn/following{/other_user}",
|
||||
"gists_url": "https://api.github.com/users/hohn/gists{/gist_id}",
|
||||
"starred_url": "https://api.github.com/users/hohn/starred{/owner}{/repo}",
|
||||
"subscriptions_url": "https://api.github.com/users/hohn/subscriptions",
|
||||
"organizations_url": "https://api.github.com/users/hohn/orgs",
|
||||
"repos_url": "https://api.github.com/users/hohn/repos",
|
||||
"events_url": "https://api.github.com/users/hohn/events{/privacy}",
|
||||
"received_events_url": "https://api.github.com/users/hohn/received_events",
|
||||
"type": "User",
|
||||
"site_admin": true
|
||||
},
|
||||
"html_url": "https://github.com/hohn/mirva-controller",
|
||||
"description": "Controller for MRVA",
|
||||
"fork": false,
|
||||
"url": "https://api.github.com/repos/hohn/mirva-controller",
|
||||
"forks_url": "https://api.github.com/repos/hohn/mirva-controller/forks",
|
||||
"keys_url": "https://api.github.com/repos/hohn/mirva-controller/keys{/key_id}",
|
||||
"collaborators_url": "https://api.github.com/repos/hohn/mirva-controller/collaborators{/collaborator}",
|
||||
"teams_url": "https://api.github.com/repos/hohn/mirva-controller/teams",
|
||||
"hooks_url": "https://api.github.com/repos/hohn/mirva-controller/hooks",
|
||||
"issue_events_url": "https://api.github.com/repos/hohn/mirva-controller/issues/events{/number}",
|
||||
"events_url": "https://api.github.com/repos/hohn/mirva-controller/events",
|
||||
"assignees_url": "https://api.github.com/repos/hohn/mirva-controller/assignees{/user}",
|
||||
"branches_url": "https://api.github.com/repos/hohn/mirva-controller/branches{/branch}",
|
||||
"tags_url": "https://api.github.com/repos/hohn/mirva-controller/tags",
|
||||
"blobs_url": "https://api.github.com/repos/hohn/mirva-controller/git/blobs{/sha}",
|
||||
"git_tags_url": "https://api.github.com/repos/hohn/mirva-controller/git/tags{/sha}",
|
||||
"git_refs_url": "https://api.github.com/repos/hohn/mirva-controller/git/refs{/sha}",
|
||||
"trees_url": "https://api.github.com/repos/hohn/mirva-controller/git/trees{/sha}",
|
||||
"statuses_url": "https://api.github.com/repos/hohn/mirva-controller/statuses/{sha}",
|
||||
"languages_url": "https://api.github.com/repos/hohn/mirva-controller/languages",
|
||||
"stargazers_url": "https://api.github.com/repos/hohn/mirva-controller/stargazers",
|
||||
"contributors_url": "https://api.github.com/repos/hohn/mirva-controller/contributors",
|
||||
"subscribers_url": "https://api.github.com/repos/hohn/mirva-controller/subscribers",
|
||||
"subscription_url": "https://api.github.com/repos/hohn/mirva-controller/subscription",
|
||||
"commits_url": "https://api.github.com/repos/hohn/mirva-controller/commits{/sha}",
|
||||
"git_commits_url": "https://api.github.com/repos/hohn/mirva-controller/git/commits{/sha}",
|
||||
"comments_url": "https://api.github.com/repos/hohn/mirva-controller/comments{/number}",
|
||||
"issue_comment_url": "https://api.github.com/repos/hohn/mirva-controller/issues/comments{/number}",
|
||||
"contents_url": "https://api.github.com/repos/hohn/mirva-controller/contents/{+path}",
|
||||
"compare_url": "https://api.github.com/repos/hohn/mirva-controller/compare/{base}...{head}",
|
||||
"merges_url": "https://api.github.com/repos/hohn/mirva-controller/merges",
|
||||
"archive_url": "https://api.github.com/repos/hohn/mirva-controller/{archive_format}{/ref}",
|
||||
"downloads_url": "https://api.github.com/repos/hohn/mirva-controller/downloads",
|
||||
"issues_url": "https://api.github.com/repos/hohn/mirva-controller/issues{/number}",
|
||||
"pulls_url": "https://api.github.com/repos/hohn/mirva-controller/pulls{/number}",
|
||||
"milestones_url": "https://api.github.com/repos/hohn/mirva-controller/milestones{/number}",
|
||||
"notifications_url": "https://api.github.com/repos/hohn/mirva-controller/notifications{?since,all,participating}",
|
||||
"labels_url": "https://api.github.com/repos/hohn/mirva-controller/labels{/name}",
|
||||
"releases_url": "https://api.github.com/repos/hohn/mirva-controller/releases{/id}",
|
||||
"deployments_url": "https://api.github.com/repos/hohn/mirva-controller/deployments"
|
||||
},
|
||||
"actor": {
|
||||
"login": "hohn",
|
||||
"id": 2253228,
|
||||
"node_id": "MDQ6VXNlcjIyNTMyMjg=",
|
||||
"avatar_url": "https://avatars.githubusercontent.com/u/2253228?v=4",
|
||||
"gravatar_id": "",
|
||||
"url": "https://api.github.com/users/hohn",
|
||||
"html_url": "https://github.com/hohn",
|
||||
"followers_url": "https://api.github.com/users/hohn/followers",
|
||||
"following_url": "https://api.github.com/users/hohn/following{/other_user}",
|
||||
"gists_url": "https://api.github.com/users/hohn/gists{/gist_id}",
|
||||
"starred_url": "https://api.github.com/users/hohn/starred{/owner}{/repo}",
|
||||
"subscriptions_url": "https://api.github.com/users/hohn/subscriptions",
|
||||
"organizations_url": "https://api.github.com/users/hohn/orgs",
|
||||
"repos_url": "https://api.github.com/users/hohn/repos",
|
||||
"events_url": "https://api.github.com/users/hohn/events{/privacy}",
|
||||
"received_events_url": "https://api.github.com/users/hohn/received_events",
|
||||
"type": "User",
|
||||
"site_admin": true
|
||||
},
|
||||
"query_language": "cpp",
|
||||
"query_pack_url": "https://objects-origin.githubusercontent.com/codeql-query-console/variant_analyses/8985/query_pack?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=queryconsoleprod%2F20240208%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240208T232733Z&X-Amz-Expires=3600&X-Amz-Signature=8eb6f15e173820073ab97b714121d8e7be62f2e04f2c4cd2192b36b252c8aaaa&X-Amz-SignedHeaders=host",
|
||||
"created_at": "2024-02-08T22:54:21Z",
|
||||
"updated_at": "2024-02-08T22:54:22Z",
|
||||
"actions_workflow_run_id": 7837027089,
|
||||
"completed_at": "2024-02-08T22:55:02Z",
|
||||
"status": "succeeded",
|
||||
"scanned_repositories": [
|
||||
{
|
||||
"repository": {
|
||||
"id": 19953044,
|
||||
"name": "flatbuffers",
|
||||
"full_name": "google/flatbuffers",
|
||||
"private": false,
|
||||
"stargazers_count": 21718,
|
||||
"updated_at": "2024-02-08T13:40:55Z"
|
||||
},
|
||||
"analysis_status": "succeeded",
|
||||
"result_count": 1,
|
||||
"artifact_size_in_bytes": 1325
|
||||
}
|
||||
],
|
||||
"skipped_repositories": {
|
||||
"access_mismatch_repos": {
|
||||
"repository_count": 0,
|
||||
"repositories": []
|
||||
},
|
||||
"not_found_repos": {
|
||||
"repository_count": 0,
|
||||
"repository_full_names": []
|
||||
},
|
||||
"no_codeql_db_repos": {
|
||||
"repository_count": 0,
|
||||
"repositories": []
|
||||
},
|
||||
"over_limit_repos": {
|
||||
"repository_count": 0,
|
||||
"repositories": []
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Run name: mirva-session-10
|
||||
Status: succeeded
|
||||
Total runs: 1
|
||||
Total successful scans: 1
|
||||
Total failed scans: 0
|
||||
Total skipped repositories: 0
|
||||
Total skipped repositories due to access mismatch: 0
|
||||
Total skipped repositories due to not found: 0
|
||||
Total skipped repositories due to no database: 0
|
||||
Total skipped repositories due to over limit: 0
|
||||
Total repositories with findings: 1
|
||||
Total findings: 1
|
||||
Repositories with findings:
|
||||
google/flatbuffers (cpp-flatbuffer-func): 1
|
||||
@@ -25,6 +25,63 @@ Compiling and bundling the QLPack (This may take a while)
|
||||
Generated encoded bundle for /Users/hohn/local/gh-mrva/FlatBuffersFunc.ql (cpp-flatbuffer-func)
|
||||
2024/02/08 14:54:14 >> POST https://api.github.com/repos/hohn/mirva-controller/code-scanning/codeql/variant-analyses
|
||||
2024/02/08 14:54:14 >> Request body: {"repositories":["google/flatbuffers"],"language":"cpp","query_pack":"H4sIAAAA...","action_repo_ref":"main"}
|
||||
|
||||
from
|
||||
body := struct {
|
||||
Repositories []string `json:"repositories"`
|
||||
Language string `json:"language"`
|
||||
Pack string `json:"query_pack"`
|
||||
Ref string `json:"action_repo_ref"`
|
||||
}
|
||||
from
|
||||
encodedBundle
|
||||
from
|
||||
id, err := utils.SubmitRun(controller, language, chunk, encodedBundle)
|
||||
from
|
||||
encodedBundle, queryId, err := utils.GenerateQueryPack(query, language, additionalPacks)
|
||||
from
|
||||
bundlePath := filepath.Join(filepath.Dir(queryPackDir),
|
||||
fmt.Sprintf("qlpack-%s-generated.tgz", uuid.New().String()))
|
||||
...
|
||||
args = []string{"pack", "bundle", "-o", bundlePath, queryPackDir}
|
||||
...
|
||||
bundleBase64 := base64.StdEncoding.EncodeToString(bundleBytes)
|
||||
return bundleBase64, queryId, nil
|
||||
So it's a base64 encoded gzipped tar file.
|
||||
In reverse: base64 -d < foo1 | gunzip | tar t
|
||||
|
||||
"query_pack":... is
|
||||
0:$ base64 -d < foo1 | gunzip | tar t|head -20
|
||||
FlatBuffersFunc.ql
|
||||
LICENSE
|
||||
qlpack.yml
|
||||
codeql-pack.lock.yml
|
||||
README.md
|
||||
.codeql/libraries/codeql/dataflow/0.1.5/qlpack.yml
|
||||
.codeql/libraries/codeql/dataflow/0.1.5/CHANGELOG.md
|
||||
.codeql/libraries/codeql/dataflow/0.1.5/codeql/dataflow/TaintTracking.qll
|
||||
.codeql/libraries/codeql/dataflow/0.1.5/codeql/dataflow/test/InlineFlowTest.qll
|
||||
.codeql/libraries/codeql/dataflow/0.1.5/codeql/dataflow/internal/DataFlowImplConsistency.qll
|
||||
.codeql/libraries/codeql/dataflow/0.1.5/codeql/dataflow/internal/DataFlowImpl.qll
|
||||
.codeql/libraries/codeql/dataflow/0.1.5/codeql/dataflow/internal/DataFlowImplCommon.qll
|
||||
.codeql/libraries/codeql/dataflow/0.1.5/codeql/dataflow/VariableCapture.qll
|
||||
.codeql/libraries/codeql/dataflow/0.1.5/codeql/dataflow/DataFlow.qll
|
||||
.codeql/libraries/codeql/tutorial/0.2.5/qlpack.yml
|
||||
.codeql/libraries/codeql/tutorial/0.2.5/CHANGELOG.md
|
||||
.codeql/libraries/codeql/tutorial/0.2.5/tutorial.qll
|
||||
.codeql/libraries/codeql/util/0.2.5/qlpack.yml
|
||||
.codeql/libraries/codeql/util/0.2.5/CHANGELOG.md
|
||||
.codeql/libraries/codeql/util/0.2.5/codeql/util/Location.qll
|
||||
...
|
||||
|
||||
0:$ base64 -d < foo1 | gunzip | tar t| tail -5
|
||||
.codeql/precompiled/271765fqs2kb5d2cdumjptd98s9.bytes.qlx
|
||||
.codeql/precompiled/629134s1rogmnmbe22m1da10kke.bytes.qlx
|
||||
.codeql/precompiled/b4c9a2bili3cn98q4a64ldecog2.bytes.qlx
|
||||
.codeql/precompiled/09f73besrcbo1ghd85ondjlsv80.bytes.qlx
|
||||
FlatBuffersFunc.qlx
|
||||
|
||||
|
||||
2024/02/08 14:54:17 << 201 https://api.github.com/repos/hohn/mirva-controller/code-scanning/codeql/variant-analyses
|
||||
2024/02/08 14:54:17 >> Response body: {
|
||||
"id": 8985,
|
||||
|
||||
50
main.go
50
main.go
@@ -49,17 +49,24 @@ func main() {
|
||||
func LogRequestDump(req *http.Request) {
|
||||
log.Printf(">> %s %s", req.Method, req.URL)
|
||||
|
||||
buf, err := io.ReadAll(req.Body)
|
||||
if err != nil {
|
||||
var w http.ResponseWriter
|
||||
log.Printf("Error reading request body: %v", err.Error())
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
log.Printf(">> Request body: %v", string(buf))
|
||||
// TODO: as function
|
||||
// TODO: show index for pk zip archives
|
||||
// TODO: show json ?toc? for
|
||||
// 2024/02/08 14:54:14 >> Request body: {"repositories":["google/flatbuffers"],
|
||||
// "language":"cpp","query_pack":"H4sIAAAA...","action_repo_ref":"main"}
|
||||
if req.Body != nil {
|
||||
buf, err := io.ReadAll(req.Body)
|
||||
if err != nil {
|
||||
var w http.ResponseWriter
|
||||
log.Printf("Error reading request body: %v", err.Error())
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
log.Printf(">> Request body: %v", string(buf))
|
||||
|
||||
reader := io.NopCloser(bytes.NewBuffer(buf))
|
||||
req.Body = reader
|
||||
reader := io.NopCloser(bytes.NewBuffer(buf))
|
||||
req.Body = reader
|
||||
}
|
||||
}
|
||||
|
||||
type contextKey struct {
|
||||
@@ -77,16 +84,17 @@ func LogResponseDump(resp *http.Response) {
|
||||
log.Printf("<< %d %s", resp.StatusCode, resp.Request.URL)
|
||||
}
|
||||
|
||||
buf, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
var w http.ResponseWriter
|
||||
log.Printf("Error reading response body: %v", err.Error())
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
if resp.Body != nil {
|
||||
buf, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
var w http.ResponseWriter
|
||||
log.Printf("Error reading response body: %v", err.Error())
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
log.Printf(">> Response body: %v", string(buf))
|
||||
|
||||
reader := io.NopCloser(bytes.NewBuffer(buf))
|
||||
resp.Body = reader
|
||||
}
|
||||
log.Printf(">> Response body: %v", string(buf))
|
||||
|
||||
reader := io.NopCloser(bytes.NewBuffer(buf))
|
||||
resp.Body = reader
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user