Unpack the request/response payload contents in the shell and keep annotated logs

README.org

@@ -193,26 +193,35 @@
 
      ./gh-mrva -h
 
-     # Submit the mrva job
+     # In log-submit-the-mrva-job.log after edit
-     SN=10
+     SN=11
      ./gh-mrva submit --language cpp --session mirva-session-$SN     \
                --list mirva-list                                     \
                --query /Users/hohn/local/gh-mrva/FlatBuffersFunc.ql >& log-$SN.out &
      tail -f log-$SN.out
 
-
      # Check the status
      ./gh-mrva status --session mirva-session-$SN |& tee log-$SN-status.out
 
-     # Download the sarif files when finished
+     # Download the sarif files and CodeQL dbs when finished
-     gh mrva download --session mirva-session-$SN \
+     ./gh-mrva download --session mirva-session-$SN \
-        --output-dir mirva-session-$SN-sarif
-
-     # Or download the sarif files and CodeQL dbs when finished
-     gh mrva download --session mirva-session-$SN \
         --download-dbs \
-        --output-dir mirva-session-$SN-sarif
+        --output-dir mirva-session-$SN-sarif \
+        |& tee log-download.log
+     # 2024/02/08 15:33:39 >> Response body is 
+     # Zip archive data, at least v1.0 to extract, compression method=deflate
+     # 0:$ unzip -v foo
+     # Archive:  foo
+     #  Length   Method    Size  Cmpr    Date    Time   CRC-32   Name
+     # --------  ------  ------- ---- ---------- ----- --------  ----
+     #     2297  Defl:N      980  57% 02-08-2024 22:54 9465f5ff  results.sarif
+     #      148  Defl:N      121  18% 02-08-2024 22:54 dc8df445  results.bqrs
+     # --------          -------  ---                            -------
+     #     2445             1101  55%                            2 files
 
+     # 0	4	Local file header signature = 0x04034b50 (PK♥♦ or "PK\3\4") PK
+     # 0x50, 0x4b, 0x03, 0x04
+     #  byteArray := []byte{ 0x50, 0x4b, 0x03, 0x04 }
    #+END_SRC
 
 ** Use the delve debugger to find sigsev

log-status.log

@@ -0,0 +1,151 @@
+# Check the status
+./gh-mrva status --session mirva-session-$SN |& tee log-$SN-status.out
+
+
+2024/02/08 15:27:29 >> GET https://api.github.com/repos/hohn/mirva-controller/code-scanning/codeql/variant-analyses/8985
+2024/02/08 15:27:29 << 200 https://api.github.com/repos/hohn/mirva-controller/code-scanning/codeql/variant-analyses/8985
+2024/02/08 15:27:29 >> Response body: {
+  "id": 8985,
+  "controller_repo": {
+    "id": 747492529,
+    "node_id": "R_kgDOLI3UsQ",
+    "name": "mirva-controller",
+    "full_name": "hohn/mirva-controller",
+    "private": false,
+    "owner": {
+      "login": "hohn",
+      "id": 2253228,
+      "node_id": "MDQ6VXNlcjIyNTMyMjg=",
+      "avatar_url": "https://avatars.githubusercontent.com/u/2253228?v=4",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/hohn",
+      "html_url": "https://github.com/hohn",
+      "followers_url": "https://api.github.com/users/hohn/followers",
+      "following_url": "https://api.github.com/users/hohn/following{/other_user}",
+      "gists_url": "https://api.github.com/users/hohn/gists{/gist_id}",
+      "starred_url": "https://api.github.com/users/hohn/starred{/owner}{/repo}",
+      "subscriptions_url": "https://api.github.com/users/hohn/subscriptions",
+      "organizations_url": "https://api.github.com/users/hohn/orgs",
+      "repos_url": "https://api.github.com/users/hohn/repos",
+      "events_url": "https://api.github.com/users/hohn/events{/privacy}",
+      "received_events_url": "https://api.github.com/users/hohn/received_events",
+      "type": "User",
+      "site_admin": true
+    },
+    "html_url": "https://github.com/hohn/mirva-controller",
+    "description": "Controller for MRVA",
+    "fork": false,
+    "url": "https://api.github.com/repos/hohn/mirva-controller",
+    "forks_url": "https://api.github.com/repos/hohn/mirva-controller/forks",
+    "keys_url": "https://api.github.com/repos/hohn/mirva-controller/keys{/key_id}",
+    "collaborators_url": "https://api.github.com/repos/hohn/mirva-controller/collaborators{/collaborator}",
+    "teams_url": "https://api.github.com/repos/hohn/mirva-controller/teams",
+    "hooks_url": "https://api.github.com/repos/hohn/mirva-controller/hooks",
+    "issue_events_url": "https://api.github.com/repos/hohn/mirva-controller/issues/events{/number}",
+    "events_url": "https://api.github.com/repos/hohn/mirva-controller/events",
+    "assignees_url": "https://api.github.com/repos/hohn/mirva-controller/assignees{/user}",
+    "branches_url": "https://api.github.com/repos/hohn/mirva-controller/branches{/branch}",
+    "tags_url": "https://api.github.com/repos/hohn/mirva-controller/tags",
+    "blobs_url": "https://api.github.com/repos/hohn/mirva-controller/git/blobs{/sha}",
+    "git_tags_url": "https://api.github.com/repos/hohn/mirva-controller/git/tags{/sha}",
+    "git_refs_url": "https://api.github.com/repos/hohn/mirva-controller/git/refs{/sha}",
+    "trees_url": "https://api.github.com/repos/hohn/mirva-controller/git/trees{/sha}",
+    "statuses_url": "https://api.github.com/repos/hohn/mirva-controller/statuses/{sha}",
+    "languages_url": "https://api.github.com/repos/hohn/mirva-controller/languages",
+    "stargazers_url": "https://api.github.com/repos/hohn/mirva-controller/stargazers",
+    "contributors_url": "https://api.github.com/repos/hohn/mirva-controller/contributors",
+    "subscribers_url": "https://api.github.com/repos/hohn/mirva-controller/subscribers",
+    "subscription_url": "https://api.github.com/repos/hohn/mirva-controller/subscription",
+    "commits_url": "https://api.github.com/repos/hohn/mirva-controller/commits{/sha}",
+    "git_commits_url": "https://api.github.com/repos/hohn/mirva-controller/git/commits{/sha}",
+    "comments_url": "https://api.github.com/repos/hohn/mirva-controller/comments{/number}",
+    "issue_comment_url": "https://api.github.com/repos/hohn/mirva-controller/issues/comments{/number}",
+    "contents_url": "https://api.github.com/repos/hohn/mirva-controller/contents/{+path}",
+    "compare_url": "https://api.github.com/repos/hohn/mirva-controller/compare/{base}...{head}",
+    "merges_url": "https://api.github.com/repos/hohn/mirva-controller/merges",
+    "archive_url": "https://api.github.com/repos/hohn/mirva-controller/{archive_format}{/ref}",
+    "downloads_url": "https://api.github.com/repos/hohn/mirva-controller/downloads",
+    "issues_url": "https://api.github.com/repos/hohn/mirva-controller/issues{/number}",
+    "pulls_url": "https://api.github.com/repos/hohn/mirva-controller/pulls{/number}",
+    "milestones_url": "https://api.github.com/repos/hohn/mirva-controller/milestones{/number}",
+    "notifications_url": "https://api.github.com/repos/hohn/mirva-controller/notifications{?since,all,participating}",
+    "labels_url": "https://api.github.com/repos/hohn/mirva-controller/labels{/name}",
+    "releases_url": "https://api.github.com/repos/hohn/mirva-controller/releases{/id}",
+    "deployments_url": "https://api.github.com/repos/hohn/mirva-controller/deployments"
+  },
+  "actor": {
+    "login": "hohn",
+    "id": 2253228,
+    "node_id": "MDQ6VXNlcjIyNTMyMjg=",
+    "avatar_url": "https://avatars.githubusercontent.com/u/2253228?v=4",
+    "gravatar_id": "",
+    "url": "https://api.github.com/users/hohn",
+    "html_url": "https://github.com/hohn",
+    "followers_url": "https://api.github.com/users/hohn/followers",
+    "following_url": "https://api.github.com/users/hohn/following{/other_user}",
+    "gists_url": "https://api.github.com/users/hohn/gists{/gist_id}",
+    "starred_url": "https://api.github.com/users/hohn/starred{/owner}{/repo}",
+    "subscriptions_url": "https://api.github.com/users/hohn/subscriptions",
+    "organizations_url": "https://api.github.com/users/hohn/orgs",
+    "repos_url": "https://api.github.com/users/hohn/repos",
+    "events_url": "https://api.github.com/users/hohn/events{/privacy}",
+    "received_events_url": "https://api.github.com/users/hohn/received_events",
+    "type": "User",
+    "site_admin": true
+  },
+  "query_language": "cpp",
+  "query_pack_url": "https://objects-origin.githubusercontent.com/codeql-query-console/variant_analyses/8985/query_pack?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=queryconsoleprod%2F20240208%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240208T232733Z&X-Amz-Expires=3600&X-Amz-Signature=8eb6f15e173820073ab97b714121d8e7be62f2e04f2c4cd2192b36b252c8aaaa&X-Amz-SignedHeaders=host",
+  "created_at": "2024-02-08T22:54:21Z",
+  "updated_at": "2024-02-08T22:54:22Z",
+  "actions_workflow_run_id": 7837027089,
+  "completed_at": "2024-02-08T22:55:02Z",
+  "status": "succeeded",
+  "scanned_repositories": [
+    {
+      "repository": {
+        "id": 19953044,
+        "name": "flatbuffers",
+        "full_name": "google/flatbuffers",
+        "private": false,
+        "stargazers_count": 21718,
+        "updated_at": "2024-02-08T13:40:55Z"
+      },
+      "analysis_status": "succeeded",
+      "result_count": 1,
+      "artifact_size_in_bytes": 1325
+    }
+  ],
+  "skipped_repositories": {
+    "access_mismatch_repos": {
+      "repository_count": 0,
+      "repositories": []
+    },
+    "not_found_repos": {
+      "repository_count": 0,
+      "repository_full_names": []
+    },
+    "no_codeql_db_repos": {
+      "repository_count": 0,
+      "repositories": []
+    },
+    "over_limit_repos": {
+      "repository_count": 0,
+      "repositories": []
+    }
+  }
+}
+
+Run name: mirva-session-10
+Status: succeeded
+Total runs: 1
+Total successful scans: 1
+Total failed scans: 0
+Total skipped repositories: 0
+Total skipped repositories due to access mismatch: 0
+Total skipped repositories due to not found: 0
+Total skipped repositories due to no database: 0
+Total skipped repositories due to over limit: 0
+Total repositories with findings: 1
+Total findings: 1
+Repositories with findings:
+  google/flatbuffers (cpp-flatbuffer-func): 1

log-submit-the-mrva-job.log

@@ -25,6 +25,63 @@ Compiling and bundling the QLPack (This may take a while)
 Generated encoded bundle for /Users/hohn/local/gh-mrva/FlatBuffersFunc.ql (cpp-flatbuffer-func)
 2024/02/08 14:54:14 >> POST https://api.github.com/repos/hohn/mirva-controller/code-scanning/codeql/variant-analyses
 2024/02/08 14:54:14 >> Request body: {"repositories":["google/flatbuffers"],"language":"cpp","query_pack":"H4sIAAAA...","action_repo_ref":"main"}
+
+from
+        body := struct {
+                        Repositories []string `json:"repositories"`
+                        Language     string   `json:"language"`
+                        Pack         string   `json:"query_pack"`
+                        Ref          string   `json:"action_repo_ref"`
+                }
+from
+        encodedBundle
+from
+        id, err := utils.SubmitRun(controller, language, chunk, encodedBundle)
+from
+        encodedBundle, queryId, err := utils.GenerateQueryPack(query, language, additionalPacks)
+from
+        bundlePath := filepath.Join(filepath.Dir(queryPackDir),
+        fmt.Sprintf("qlpack-%s-generated.tgz", uuid.New().String()))  
+        ...
+        args = []string{"pack", "bundle", "-o", bundlePath, queryPackDir}
+        ...
+	bundleBase64 := base64.StdEncoding.EncodeToString(bundleBytes)
+	return bundleBase64, queryId, nil
+So it's a base64 encoded gzipped tar file.
+In reverse:  base64 -d < foo1 | gunzip | tar t
+
+"query_pack":... is
+0:$ base64 -d < foo1 | gunzip | tar t|head -20
+FlatBuffersFunc.ql
+LICENSE
+qlpack.yml
+codeql-pack.lock.yml
+README.md
+.codeql/libraries/codeql/dataflow/0.1.5/qlpack.yml
+.codeql/libraries/codeql/dataflow/0.1.5/CHANGELOG.md
+.codeql/libraries/codeql/dataflow/0.1.5/codeql/dataflow/TaintTracking.qll
+.codeql/libraries/codeql/dataflow/0.1.5/codeql/dataflow/test/InlineFlowTest.qll
+.codeql/libraries/codeql/dataflow/0.1.5/codeql/dataflow/internal/DataFlowImplConsistency.qll
+.codeql/libraries/codeql/dataflow/0.1.5/codeql/dataflow/internal/DataFlowImpl.qll
+.codeql/libraries/codeql/dataflow/0.1.5/codeql/dataflow/internal/DataFlowImplCommon.qll
+.codeql/libraries/codeql/dataflow/0.1.5/codeql/dataflow/VariableCapture.qll
+.codeql/libraries/codeql/dataflow/0.1.5/codeql/dataflow/DataFlow.qll
+.codeql/libraries/codeql/tutorial/0.2.5/qlpack.yml
+.codeql/libraries/codeql/tutorial/0.2.5/CHANGELOG.md
+.codeql/libraries/codeql/tutorial/0.2.5/tutorial.qll
+.codeql/libraries/codeql/util/0.2.5/qlpack.yml
+.codeql/libraries/codeql/util/0.2.5/CHANGELOG.md
+.codeql/libraries/codeql/util/0.2.5/codeql/util/Location.qll
+...
+
+0:$ base64 -d < foo1 | gunzip | tar t| tail -5
+.codeql/precompiled/271765fqs2kb5d2cdumjptd98s9.bytes.qlx
+.codeql/precompiled/629134s1rogmnmbe22m1da10kke.bytes.qlx
+.codeql/precompiled/b4c9a2bili3cn98q4a64ldecog2.bytes.qlx
+.codeql/precompiled/09f73besrcbo1ghd85ondjlsv80.bytes.qlx
+FlatBuffersFunc.qlx
+
+
 2024/02/08 14:54:17 << 201 https://api.github.com/repos/hohn/mirva-controller/code-scanning/codeql/variant-analyses
 2024/02/08 14:54:17 >> Response body: {
   "id": 8985,

main.go

@@ -49,6 +49,12 @@ func main() {
 func LogRequestDump(req *http.Request) {
 	log.Printf(">> %s %s", req.Method, req.URL)
 
+	// TODO: as function
+	// TODO: show index for pk zip archives
+	// TODO: show json ?toc? for
+	//   2024/02/08 14:54:14 >> Request body: {"repositories":["google/flatbuffers"],
+	//   "language":"cpp","query_pack":"H4sIAAAA...","action_repo_ref":"main"}
+	if req.Body != nil {
 		buf, err := io.ReadAll(req.Body)
 		if err != nil {
 			var w http.ResponseWriter
@@ -61,6 +67,7 @@ func LogRequestDump(req *http.Request) {
 		reader := io.NopCloser(bytes.NewBuffer(buf))
 		req.Body = reader
 	}
+}
 
 type contextKey struct {
 	name string
@@ -77,6 +84,7 @@ func LogResponseDump(resp *http.Response) {
 		log.Printf("<< %d %s", resp.StatusCode, resp.Request.URL)
 	}
 
+	if resp.Body != nil {
 		buf, err := io.ReadAll(resp.Body)
 		if err != nil {
 			var w http.ResponseWriter
@@ -88,5 +96,5 @@ func LogResponseDump(resp *http.Response) {
 
 		reader := io.NopCloser(bytes.NewBuffer(buf))
 		resp.Body = reader
-
+	}
 }