fix test configuration

Switch
client, err := gh.HTTPClient(nil)
to
client := &http.Client{}
and everything works again

.tool-versions

@@ -0,0 +1 @@
+golang 1.22.3

FlatBuffersFunc.ql

@@ -1,6 +1,6 @@
 /**
  * @name pickfun
- * @description pick function from FlatBuffers
+ * @description Pick function from FlatBuffers
  * @kind problem
  * @id cpp-flatbuffer-func
  * @problem.severity warning

Fprintf.ql

@@ -0,0 +1,14 @@
+/**
+ * @name findPrintf
+ * @description Find calls to plain fprintf
+ * @kind problem
+ * @id cpp-fprintf-call
+ * @problem.severity warning
+ */
+
+import cpp
+
+from FunctionCall fc
+where
+  fc.getTarget().getName() = "fprintf"
+select fc, "call of fprintf"

README.org

@@ -104,6 +104,7 @@
         cd ~/local/gh-mrva
         # Build it
         go mod edit -replace="github.com/GitHubSecurityLab/gh-mrva=/Users/hohn/local/gh-mrva"
+
         go build .
 
         # Install 
@@ -127,17 +128,17 @@
        # git checkout codeql-cli/v2.15.5
        codeql_path: /Users/hohn/local/codeql-lib
        controller: hohn/mirva-controller
-       list_file: /Users/hohn/local/gh-mrva/mirva-list-databases.json
+       list_file: /Users/hohn/work-gh/mrva/gh-mrva/mirva-list-databases.json
 
        eof
      #+END_SRC
 
    3. Submit the mrva job
       #+BEGIN_SRC sh 
-        cd ~/local/gh-mrva
-        ./gh-mrva submit --language cpp --session mirva-session-73 \
+        cd ~/work-gh/mrva/gh-mrva/
+        ./gh-mrva submit --language cpp --session mirva-session-200 \
            --list mirva-list                                       \
-           --query /Users/hohn/local/gh-mrva/FlatBuffersFunc.ql
+           --query ~/work-gh/mrva/gh-mrva/FlatBuffersFunc.ql
       #+END_SRC
 
    4. Check the status and download the sarif files
@@ -157,6 +158,7 @@
            --output-dir mirva-session-73
       #+END_SRC
 
+** curl checks for mrva server
 * Miscellaneous Notes
 ** Action logs on Controller Repository
    The action logs are on the controller repository at
@@ -189,62 +191,68 @@
    https://github.com/github/codeql-variant-analysis-action/blob/main/variant-analysis-workflow.yml
 ** Compacted Edit-Run-Debug Cycle
    With a full [[*Using MRVA][Using MRVA]] cycle done, only these steps are needed in a
-   edit-run-debug cycle.
+   edit-run-debug cycle.  Note that paths must be updated for your system.
    #+BEGIN_SRC sh 
-     cd ~/local/gh-mrva
+     # Build the client
+     cd ~/work-gh/mrva/gh-mrva
 
-     # Build it
      go clean
      go build . # go build -gcflags="all=-N -l" . 
 
      ./gh-mrva -h
 
-     # In log-submit-the-mrva-job.log after edit
-     SN=106
-     ./gh-mrva submit --language cpp --session mirva-session-$SN     \
-               --list mirva-list                                     \
-               --query /Users/hohn/local/gh-mrva/FlatBuffersFunc.ql >& log-submit-$SN.log &
-     sleep 1 && em log-submit-$SN.log
+     # Set up the configuration -- check your paths
+     cat > ~/.config/gh-mrva/config.yml <<eof
+            # The following options are supported
+            # codeql_path: Path to CodeQL distribution (checkout of codeql repo)
+            # controller: NWO of the MRVA controller to use
+            # list_file: Path to the JSON file containing the target repos
 
+            # git checkout codeql-cli/v2.15.5
+            codeql_path: /Users/hohn/local/codeql-lib
+            controller: hohn/mirva-controller
+            list_file: /Users/hohn/work-gh/mrva/gh-mrva/mirva-list-databases.json
+
+     eof
+     
+     # Define utility functions
      submit (){
          SN=$1
-         cd ~/local/gh-mrva
+         cd ~/work-gh/mrva/gh-mrva
          ./gh-mrva submit --language cpp --session mirva-session-$SN     \
                    --list mirva-list                                     \
-                   --query /Users/hohn/local/gh-mrva/FlatBuffersFunc.ql >& log-submit-$SN.log &
-         # sleep 1 && em log-submit-$SN.log
+                   --query /Users/hohn/work-gh/mrva/gh-mrva/FlatBuffersFunc.ql >& log-submit-$SN.log &
+         sleep 1 && em log-submit-$SN.log
      }
-     # submit 173
 
-     # Check the status
      sessstatus (){
          SN=$1
-         cd ~/local/gh-mrva
+         cd ~/work-gh/mrva/gh-mrva
          ./gh-mrva status --session mirva-session-$SN >& log-$SN-status.log &
          sleep 1 && em log-$SN-status.log
      }
-     # sessstatus 191
 
      # Download the sarif files and CodeQL dbs when finished
      dl (){
          SN=$1
-         cd ~/local/gh-mrva
+         cd ~/work-gh/mrva/gh-mrva
          ./gh-mrva download --session mirva-session-$SN \
                    --download-dbs \
                    --output-dir mirva-session-$SN-sarif \
                    >& log-download-$SN.log &
          sleep 1 && em log-download-$SN.log 
      }
+
      # Just download sarif / bqrs zip file
      dl (){
          SN=$1
-         cd ~/local/gh-mrva
+         cd ~/work-gh/mrva/gh-mrva
          ./gh-mrva download --session mirva-session-$SN \
                    --output-dir mirva-session-$SN-sarif \
                    >& log-download-$SN.log &
          sleep 1 && em log-download-$SN.log 
      }
-     # dl 191
+     
 
      submit      211
      sessstatus  211
@@ -264,10 +272,16 @@
      # Type 'help' for list of commands.
      # (dlv) c
 
+     # dlv debug builds, so the above build may be redundant
      dlv debug -- download --session mirva-session-$SN \
          --download-dbs \
-         --output-dir mirva-session-$SN-sarif \
+         --output-dir mirva-session-$SN-sarif 
 
+     # dlv may say 'no sources', but this works anyay
+     b main.main    
+     l
+
+     # This inline use of dlv may fail; attaching to a process is more reliable
    #+END_SRC
 ** VS Code Debugger Configuration
 *** launch.json for download

gh-mrva-selection.json

@@ -0,0 +1,9 @@
+{
+    "mirva-list": [
+        "Serial-Studio/Serial-Studio",
+        "UEFITool/UEFITool",
+        "aircrack-ng/aircrack-ng",
+        "bulk-builder/bulk-builder",
+        "tesseract/tesseract"
+    ]
+}

go.mod

@@ -5,13 +5,13 @@ go 1.19
 require (
 	github.com/cli/go-gh v1.2.1
 	github.com/motemen/go-loghttp v0.0.0-20231107055348-29ae44b293f4
+	github.com/motemen/go-nuts v0.0.0-20220604134737-2658d0104f31
 	github.com/spf13/cobra v1.7.0
 )
 
 require (
 	github.com/aymanbagabas/go-osc52 v1.2.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/motemen/go-nuts v0.0.0-20220604134737-2658d0104f31 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 )
 
@@ -33,4 +33,4 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // direct
 )
 
-replace github.com/GitHubSecurityLab/gh-mrva => /Users/hohn/local/gh-mrva
+replace github.com/GitHubSecurityLab/gh-mrva => /Users/hohn/work-gh/mrva/gh-mrva

main.go

@@ -28,9 +28,12 @@ import (
 	"compress/gzip"
 	"encoding/base64"
 	"encoding/json"
+	"flag"
 	"io"
 	"log"
+	"log/slog"
 	"net/http"
+	"os"
 	"time"
 
 	"github.com/GitHubSecurityLab/gh-mrva/cmd"
@@ -40,6 +43,44 @@ import (
 )
 
 func main() {
+
+	helpFlag := flag.Bool("help", false, "Display help message")
+	logLevel := flag.String("loglevel", "info", "Set log level: debug, info, warn, error")
+
+	// Custom usage function for the help flag
+	flag.Usage = func() {
+		log.Printf("Usage of %s:\n", os.Args[0])
+		flag.PrintDefaults()
+		log.Println("\nExamples:")
+		log.Println("go run main.go -loglevel=debug ")
+	}
+
+	// Parse the flags
+	flag.Parse()
+
+	// Handle the help flag
+	if *helpFlag {
+		flag.Usage()
+		return
+	}
+
+	// Apply 'loglevel' flag
+	switch *logLevel {
+	case "debug":
+		slog.SetLogLoggerLevel(slog.LevelDebug)
+	case "info":
+		slog.SetLogLoggerLevel(slog.LevelInfo)
+	case "warn":
+		slog.SetLogLoggerLevel(slog.LevelWarn)
+	case "error":
+		slog.SetLogLoggerLevel(slog.LevelError)
+	default:
+		log.Printf("Invalid logging verbosity level: %s", *logLevel)
+		os.Exit(1)
+	}
+
+	log.Printf("logging verbosity level: %s", *logLevel)
+
 	var transport = &loghttp.Transport{
 		Transport:   http.DefaultTransport,
 		LogRequest:  LogRequestDump,
@@ -78,7 +119,7 @@ func IsBase64Gzip(val []byte) bool {
 }
 
 func LogRequestDump(req *http.Request) {
-	log.Printf(">> %s %s", req.Method, req.URL)
+	slog.Debug(">> %s %s", req.Method, req.URL)
 	req.Body = LogBody(req.Body, "request")
 }
 
@@ -129,7 +170,7 @@ func LogBody(body io.ReadCloser, from string) io.ReadCloser {
 		buf, err := io.ReadAll(body)
 		if err != nil {
 			var w http.ResponseWriter
-			log.Fatalf("Error reading %s body: %v", from, err.Error())
+			slog.Error("Error reading %s body: %v", from, err.Error())
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return nil
 		}
@@ -144,36 +185,36 @@ func LogBody(body io.ReadCloser, from string) io.ReadCloser {
 				// Unknown message, try pretty-printing json
 				pjson, err := PPJson(string(buf))
 				if err != nil {
-					log.Printf(">> %s body: %v", from, string(buf))
+					slog.Debug(">> %s body: %v", from, string(buf))
 				} else {
-					log.Printf(">> %s body: {\n%v\n}", from, pjson)
+					slog.Debug(">> %s body: {\n%v\n}", from, pjson)
 				}
 				goto BodyDone
 			}
 
 			// Print index for encoded query packs in the json <value>:
 			// {..."query_pack": <value>,...}
-			log.Printf(">> %s body: {\n", from)
-			log.Printf("    \"%s\": \"%s\"\n", "action_repo_ref", m.ActionRepoRef)
-			log.Printf("    \"%s\": \"%s\"\n", "language", m.Language)
+			slog.Debug(">> body: {\n", "from", from)
+			slog.Debug("    \n", "action_repo_ref", m.ActionRepoRef)
+			slog.Debug("    \n", "language", m.Language)
 			pjson, err := json.MarshalIndent(m.Repositories, "", "    ")
 			if err != nil {
-				log.Printf("    \"%s\": \"%s\"\n", "repositories", m.Repositories[:])
+				slog.Debug("    \n", "repositories", m.Repositories[:])
 			} else {
-				log.Printf("    \"%s\": %s\n", "repositories", pjson)
+				slog.Debug("    \n", "repositories", pjson)
 			}
 
 			// Provide custom logging for encoded, compressed tar file
 			if IsBase64Gzip([]byte(m.QueryPack)) {
 				LogBase64GzippedTar(m)
 			} else {
-				log.Printf("    \"%s\": \"%s\"\n", "query_pack", m.QueryPack)
+				slog.Debug("    \"%s\": \"%s\"\n", "query_pack", m.QueryPack)
 			}
 
-			log.Printf("\n}")
+			slog.Debug("\n}")
 
 		} else {
-			log.Printf(">> %s body: %v", from, string(buf))
+			slog.Debug(">> %s body: %v", from, string(buf))
 		}
 
 	BodyDone:
@@ -201,17 +242,17 @@ func LogBase64GzippedTar(m SubmitMsg) {
 	// base64 decode the body
 	data, err := base64.StdEncoding.DecodeString(m.QueryPack)
 	if err != nil {
-		log.Fatalln("body decoding error:", err)
+		slog.Error("body decoding error", "err", err)
 	}
 	// gunzip the decoded body
 	gzb := bytes.NewBuffer(data)
 	gzr, err := gzip.NewReader(gzb)
 	if err != nil {
-		log.Fatal(err)
+		slog.Error("unzip error", "err", err)
 	}
 	// tar t the gunzipped body
-	log.Printf("    \"%s\": \n", "query_pack")
-	log.Printf("        base64 encoded gzipped tar file, contents:\n")
+	slog.Debug("    \"query_pack\": \n")
+	slog.Debug("        base64 encoded gzipped tar file, contents:\n")
 	tr := tar.NewReader(gzr)
 	for {
 		hdr, err := tr.Next()
@@ -219,10 +260,10 @@ func LogBase64GzippedTar(m SubmitMsg) {
 			break // End of archive
 		}
 		if err != nil {
-			log.Fatalln("Tar listing failure:", err)
+			slog.Error("Tar listing failure", "err", err)
 		}
-		// TODO: head / tail the listing
-		log.Printf("        %s\n", hdr.Name)
+		// TODO: cli option to head / tail the listing
+		slog.Debug("        ", "", hdr.Name)
 	}
 }
 
@@ -236,11 +277,11 @@ func ShowZipIndex(buf []byte, from string) {
 	}
 
 	// Print the archive index
-	log.Printf(">> %s body:\n", from)
-	log.Printf("zip file, contents:\n")
+	slog.Debug(">> body:\n", "from", from)
+	slog.Debug("zip file, contents:\n")
 
 	for _, f := range r.File {
-		log.Printf("\t%s\n", f.Name)
+		slog.Debug("\t", f.Name)
 	}
 }
 
@@ -253,10 +294,11 @@ var ContextKeyRequestStart = &contextKey{"RequestStart"}
 func LogResponseDump(resp *http.Response) {
 	ctx := resp.Request.Context()
 	if start, ok := ctx.Value(ContextKeyRequestStart).(time.Time); ok {
-		log.Printf("<< %d %s (%s)", resp.StatusCode, resp.Request.URL,
-			roundtime.Duration(time.Since(start), 2))
+		slog.Debug("<< ", "status", resp.StatusCode,
+			"url", resp.Request.URL,
+			"duration", roundtime.Duration(time.Since(start), 2))
 	} else {
-		log.Printf("<< %d %s", resp.StatusCode, resp.Request.URL)
+		slog.Debug("<< ", "status", resp.StatusCode, "url", resp.Request.URL)
 	}
 
 	resp.Body = LogBody(resp.Body, "response")

qlpack.yml

@@ -3,4 +3,3 @@ name: codeql-dataflow-ii-cpp
 version: 0.0.1
 dependencies:
   codeql/cpp-all: 0.5.3
-  

utils/utils.go

@@ -9,6 +9,8 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"log/slog"
+	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -114,7 +116,12 @@ func GetRunDetails(controller string, runId int) (map[string]interface{}, error)
 	response := make(map[string]interface{})
 
 	// err = client.Get(fmt.Sprintf("repos/%s/code-scanning/codeql/variant-analyses/%d", controller, runId), &response)
-	err = client.Get(fmt.Sprintf("http://localhost:8080/repos/%s/code-scanning/codeql/variant-analyses/%d", controller, runId), &response)
+	url := os.Getenv("MRVA_SERVER_URL")
+	if url == "" {
+		return nil, fmt.Errorf("missing MRVA_SERVER_URL in environment")
+	}
+
+	err = client.Get(fmt.Sprintf("%s/repos/%s/code-scanning/codeql/variant-analyses/%d", url, controller, runId), &response)
 	if err != nil {
 		return nil, err
 	}
@@ -129,7 +136,12 @@ func GetRunRepositoryDetails(controller string, runId int, nwo string) (map[stri
 	}
 	response := make(map[string]interface{})
 	// err = client.Get(fmt.Sprintf("repos/%s/code-scanning/codeql/variant-analyses/%d/repos/%s", controller, runId, nwo), &response)
-	err = client.Get(fmt.Sprintf("http://localhost:8080/repos/%s/code-scanning/codeql/variant-analyses/%d/repos/%s", controller, runId, nwo), &response)
+	url := os.Getenv("MRVA_SERVER_URL")
+	if url == "" {
+		return nil, fmt.Errorf("missing MRVA_SERVER_URL in environment")
+	}
+
+	err = client.Get(fmt.Sprintf("%s/repos/%s/code-scanning/codeql/variant-analyses/%d/repos/%s", url, controller, runId, nwo), &response)
 	if err != nil {
 		return nil, err
 	}
@@ -196,7 +208,12 @@ func SubmitRun(controller string, language string, repoChunk []string, bundle st
 	}
 	response := make(map[string]interface{})
 	// err = client.Post(fmt.Sprintf("repos/%s/code-scanning/codeql/variant-analyses", controller), &buf, &response)
-	err = client.Post(fmt.Sprintf("http://localhost:8080/repos/%s/code-scanning/codeql/variant-analyses", controller), &buf, &response)
+	url := os.Getenv("MRVA_SERVER_URL")
+	if url == "" {
+		return 0, fmt.Errorf("missing MRVA_SERVER_URL in environment")
+	}
+
+	err = client.Post(fmt.Sprintf("%s/repos/%s/code-scanning/codeql/variant-analyses", url, controller), &buf, &response)
 	if err != nil {
 		return -1, err
 	}
@@ -533,72 +550,116 @@ func DownloadWorker(wg *sync.WaitGroup, taskChannel <-chan models.DownloadTask,
 }
 
 func downloadArtifact(url string, task models.DownloadTask) error {
-	client, err := gh.HTTPClient(nil)
-	if err != nil {
-		return err
+	slog.Info("downloadArtifact/start", "url", url, "task", task)
+
+	// client, err := gh.HTTPClient(nil)
+	client := &http.Client{}
+
+	// if err != nil {
+	// 	slog.Error("downloadArtifact/httpClient failed", "err", err)
+	// 	return err
+	// }
+
+	// Inspect the type
+	slog.Info("downloadArtifact/httpClient/type", "type", fmt.Sprintf("%T", client))
+
+	// Check if it has a custom Transport (e.g., for auth hooks)
+	if client.Transport != nil {
+		slog.Info("downloadArtifact/httpClient/transport", "transport_type", fmt.Sprintf("%T", client.Transport))
+
+		// If it's a round-tripper wrapper, you can often unwrap or type assert it
+		switch tr := client.Transport.(type) {
+		case *http.Transport:
+			slog.Info("downloadArtifact/httpClient/transport/http.Transport", "details", fmt.Sprintf("%+v", tr))
+		default:
+			slog.Info("downloadArtifact/httpClient/transport/unknown", "details", fmt.Sprintf("%#v", tr))
+		}
+	} else {
+		slog.Info("downloadArtifact/httpClient/transport", "transport", "nil")
 	}
+
+	// Log timeout, just in case
+	slog.Info("downloadArtifact/httpClient/timeout", "timeout", client.Timeout)
+
 	resp, err := client.Get(url)
 	if err != nil {
+		slog.Error("downloadArtifact/get", "url", url, "err", err)
 		return err
 	}
 	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
-		log.Fatal(err)
+		slog.Error("downloadArtifact/readBody", "err", err)
+		return err
 	}
 
 	zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
 	if err != nil {
-		log.Fatal(err)
+		slog.Error("downloadArtifact/newZipReader", "err", err)
+		return err
 	}
 
 	downloadedFiles := []string{}
 	for _, zf := range zipReader.File {
+		slog.Info("downloadArtifact/zipEntry", "name", zf.Name)
 
 		if zf.Name != "results.sarif" && zf.Name != "results.bqrs" {
 			continue
 		}
+
 		f, err := zf.Open()
 		if err != nil {
-			log.Fatal(err)
+			slog.Error("downloadArtifact/openZipFile", "name", zf.Name, "err", err)
+			return err
 		}
 		defer f.Close()
+
 		content, err := io.ReadAll(f)
 		if err != nil {
-			log.Fatal(err)
+			slog.Error("downloadArtifact/readZipContent", "name", zf.Name, "err", err)
+			return err
 		}
 
 		outputDir := task.OutputDir
 		outputFilename := task.OutputFilename
 		if zf.Name == "results.bqrs" {
-			outputFilename = outputFilename + ".bqrs"
+			outputFilename += ".bqrs"
 		} else if zf.Name == "results.sarif" {
-			outputFilename = outputFilename + ".sarif"
+			outputFilename += ".sarif"
 		}
 
 		// replace remote-query with real query id
 		content = bytes.Replace(content, []byte("remote-query"), []byte(task.QueryId), -1)
 
 		resultPath := filepath.Join(outputDir, outputFilename)
+		slog.Info("downloadArtifact/writeFile",
+			"outputFilename", outputFilename,
+			"resultPath", resultPath)
+
 		err = os.WriteFile(resultPath, content, os.ModePerm)
 		if err != nil {
+			slog.Error("downloadArtifact/writeFileError", "path", resultPath, "err", err)
 			return err
 		}
+
 		downloadedFiles = append(downloadedFiles, resultPath)
 	}
 
 	if len(downloadedFiles) == 0 {
-		return errors.New("No results files found in artifact")
-	} else {
-		fmt.Println("Downloaded", downloadedFiles)
-		return nil
+		err := errors.New("no results files found in artifact")
+		slog.Error("downloadArtifact/empty", "err", err)
+		return err
 	}
+
+	slog.Info("downloadArtifact/success", "files", downloadedFiles)
+	return nil
 }
 
 func DownloadResults(task models.DownloadTask) error {
 	// download artifact (BQRS or SARIF)
 	runRepositoryDetails, err := GetRunRepositoryDetails(task.Controller, task.RunId, task.Nwo)
+	slog.Info("DownloadResults", "runRepositoryDetails", runRepositoryDetails)
 	if err != nil {
 		return errors.New("Failed to get run repository details")
 	}
@@ -623,7 +684,14 @@ func DownloadDatabase(task models.DownloadTask) error {
 		return err
 	}
 	// resp, err := client.Get(fmt.Sprintf("https://api.github.com/repos/%s/code-scanning/codeql/databases/%s", task.Nwo, task.Language))
-	resp, err := client.Get(fmt.Sprintf("http://localhost:8080/repos/%s/code-scanning/codeql/databases/%s", task.Nwo, task.Language))
+	// resp, err := client.Get(fmt.Sprintf("http://localhost:8080/repos/%s/code-scanning/codeql/databases/%s", task.Nwo, task.Language))
+
+	url := os.Getenv("MRVA_SERVER_URL")
+	if url == "" {
+		return fmt.Errorf("missing MRVA_SERVER_URL in environment")
+	}
+	resp, err := client.Get(fmt.Sprintf("%s/repos/%s/code-scanning/codeql/databases/%s", url, task.Nwo, task.Language))
+
 	if err != nil {
 		return err
 	}