Initial version

.github/workflows/release.yml

@@ -0,0 +1,14 @@
+name: release
+on:
+  push:
+    tags:
+      - "v*"
+permissions:
+  contents: write
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: cli/gh-extension-precompile@v1

README.md

@@ -1 +1,29 @@
-# gh-mrva
+# gh-mrva
+
+## Configuration
+
+A configuration file will be created in `~/.config/mrva/config.yml`. The following options are supported:
+- `controller`: NWO of the MRVA controller to use
+- `listFile`: Path to the JSON file containing the target repos
+
+## Usage
+
+Until the extension gets published you can use `go run .` instead of `gh mrva`
+
+### Submit a new query
+
+```bash
+gh mrva submit [--controller <CONTROLLER>] --lang <LANGUAGE> [--list-file <LISTFILE>] --list <LIST> --query <QUERY> [--name <NAME>]
+```
+
+Note: `controller` and `list-file` are only optionals if defined in the configuration file
+Note: if a `name` (any arbitrary name) is provided, the resulting run IDs will be stored in the configuration file so they can be referenced later for download
+
+### Download the results
+
+```bash
+gh mrva download [--controller <CONTROLLER>] --lang <LANGUAGE> --output-dir <OUTPUTDIR> [--name <NAME> | --run <ID>] [--download-dbs]
+```
+
+Note: `controller` is only optionals if defined in the configuration file
+Note: if a `name` is provided, the run ID is not necessary and instead `gh-mrva` will download the artifacts associated that `name` as found in the configuration file

go.mod

@@ -0,0 +1,26 @@
+module github.com/pwntester/gh-mrva
+
+go 1.19
+
+require github.com/cli/go-gh v1.2.1
+
+require (
+	github.com/cli/safeexec v1.0.0 // indirect
+	github.com/cli/shurcooL-graphql v0.0.2 // indirect
+	github.com/google/uuid v1.3.0 // direct
+	github.com/henvic/httpretty v0.0.6 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
+	github.com/mattn/go-isatty v0.0.16 // indirect
+	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/muesli/termenv v0.12.0 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/thlib/go-timezone-local v0.0.0-20210907160436-ef149e42d28e // indirect
+	github.com/tidwall/gjson v1.14.4 // direct
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
+	golang.org/x/net v0.7.0 // indirect
+	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/term v0.5.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // direct
+)

go.sum

@@ -0,0 +1,71 @@
+github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
+github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls=
+github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E=
+github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
+github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
+github.com/cli/go-gh v1.2.1 h1:xFrjejSsgPiwXFP6VYynKWwxLQcNJy3Twbu82ZDlR/o=
+github.com/cli/go-gh v1.2.1/go.mod h1:Jxk8X+TCO4Ui/GarwY9tByWm/8zp4jJktzVZNlTW5VM=
+github.com/cli/safeexec v1.0.0 h1:0VngyaIyqACHdcMNWfo6+KdUYnqEr2Sg+bSP1pdF+dI=
+github.com/cli/safeexec v1.0.0/go.mod h1:Z/D4tTN8Vs5gXYHDCbaM1S/anmEDnJb1iW0+EJ5zx3Q=
+github.com/cli/shurcooL-graphql v0.0.2 h1:rwP5/qQQ2fM0TzkUTwtt6E2LbIYf6R+39cUXTa04NYk=
+github.com/cli/shurcooL-graphql v0.0.2/go.mod h1:tlrLmw/n5Q/+4qSvosT+9/W5zc8ZMjnJeYBxSdb4nWA=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
+github.com/henvic/httpretty v0.0.6 h1:JdzGzKZBajBfnvlMALXXMVQWxWMF/ofTy8C3/OSUTxs=
+github.com/henvic/httpretty v0.0.6/go.mod h1:X38wLjWXHkXT7r2+uK8LjCMne9rsuNaBLJ+5cU2/Pmo=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
+github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
+github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
+github.com/muesli/termenv v0.12.0 h1:KuQRUE3PgxRFWhq4gHvZtPSLCGDqM5q/cYr1pZ39ytc=
+github.com/muesli/termenv v0.12.0/go.mod h1:WCCv32tusQ/EEZ5S8oUIIrC/nIuBcxCVqlN4Xfkv+7A=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/thlib/go-timezone-local v0.0.0-20210907160436-ef149e42d28e h1:BuzhfgfWQbX0dWzYzT1zsORLnHRv3bcRcsaUk0VmXA8=
+github.com/thlib/go-timezone-local v0.0.0-20210907160436-ef149e42d28e/go.mod h1:/Tnicc6m/lsJE0irFMA0LfIwTBo4QP7A8IfyIv4zZKI=
+github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
+github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
+github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+golang.org/x/net v0.0.0-20220923203811-8be639271d50/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -0,0 +1,739 @@
+package main
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"github.com/cli/go-gh"
+	"github.com/cli/go-gh/pkg/api"
+	"github.com/google/uuid"
+	"github.com/tidwall/gjson"
+	"gopkg.in/yaml.v3"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"text/template"
+)
+
+const (
+	MAX_MRVA_REPOSITORIES = 1000
+)
+
+var (
+	configFilePath = ""
+	controller     = ""
+	language       = ""
+	runName        = ""
+	listFile       = ""
+)
+
+func registerGlobalFlags(fset *flag.FlagSet) {
+	// parsing subcommands in GoLang: https://abhinavg.net/2022/08/13/flag-subcommand/
+	flag.VisitAll(func(f *flag.Flag) {
+		fset.Var(f.Value, f.Name, f.Usage)
+	})
+}
+
+func resolveRepositories(listFile string, list string) ([]string, error) {
+	fmt.Printf("Resolving %s repositories from %s\n", list, listFile)
+	jsonFile, err := os.Open(listFile)
+	if err != nil {
+		fmt.Println(err)
+		return nil, err
+	}
+	defer jsonFile.Close()
+	byteValue, _ := ioutil.ReadAll(jsonFile)
+	json := string(byteValue)
+	repositories := gjson.Get(json, list)
+	var result []string
+	for _, x := range repositories.Array() {
+		result = append(result, x.String())
+	}
+	return result, nil
+}
+
+func findPackRoot(queryFile string) string {
+	// Starting on the directory of queryPackDir, go down until a qlpack.yml find is found. return that directory
+	// If no qlpack.yml is found, return the directory of queryFile
+	currentDir := filepath.Dir(queryFile)
+	for currentDir != "/" {
+		if _, err := os.Stat(filepath.Join(currentDir, "qlpack.yml")); errors.Is(err, os.ErrNotExist) {
+			// qlpack.yml not found, go up one level
+			currentDir = filepath.Dir(currentDir)
+		} else {
+			return currentDir
+		}
+	}
+	return filepath.Dir(queryFile)
+}
+
+func packPacklist(dir string, includeQueries bool) []string {
+	// since 2.7.1, packlist returns an object with a "paths" property that is a list of packs.
+	args := []string{"pack", "packlist", "--format=json"}
+	if !includeQueries {
+		args = append(args, "--no-include-queries")
+	}
+	args = append(args, dir)
+	json, err := exec.Command("codeql", args...).Output()
+	// parse the json
+	packlist := gjson.Get(string(json), "paths")
+	if err != nil {
+		log.Fatal(err)
+	}
+	var result []string
+	for _, x := range packlist.Array() {
+		result = append(result, x.String())
+	}
+	return result
+}
+
+func copyFile(srcPath string, targetPath string) error {
+	err := os.MkdirAll(filepath.Dir(targetPath), os.ModePerm)
+	if err != nil {
+		return err
+	}
+	bytesRead, err := ioutil.ReadFile(srcPath)
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(targetPath, bytesRead, 0644)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Fixes the qlpack.yml file to be correct in the context of the MRVA request.
+// Performs the following fixes:
+// - Updates the default suite of the query pack. This is used to ensure
+// only the specified query is run.
+// - Ensures the query pack name is set to the name expected by the server.
+// - Removes any `${workspace}` version references from the qlpack.yml file. Converts them
+// to `*` versions.
+// @param queryPackDir The directory containing the query pack
+// @param packRelativePath The relative path to the query pack from the root of the query pack
+func fixPackFile(queryPackDir string, packRelativePath string) error {
+	packPath := filepath.Join(queryPackDir, "qlpack.yml")
+	packFile, err := ioutil.ReadFile(packPath)
+	if err != nil {
+		return err
+	}
+	var packData map[string]interface{}
+	err = yaml.Unmarshal(packFile, &packData)
+	if err != nil {
+		return err
+	}
+	// update the default suite
+	defaultSuiteFile := packData["defaultSuiteFile"]
+	if defaultSuiteFile != nil {
+		// remove the defaultSuiteFile property
+		delete(packData, "defaultSuiteFile")
+	}
+	packData["defaultSuite"] = map[string]string{
+		"query":       packRelativePath,
+		"description": "Query suite for Variant Analysis",
+	}
+
+	// update the name
+	packData["name"] = "codeql-remote/query"
+
+	// remove any `${workspace}` version references
+	dependencies := packData["dependencies"]
+	if dependencies != nil {
+		// for key and value in dependencies
+		for key, value := range dependencies.(map[string]interface{}) {
+			// if value is a string and value contains `${workspace}`
+			if value == "${workspace}" {
+				// replace the value with `*`
+				packData["dependencies"].(map[string]string)[key] = "*"
+			}
+		}
+	}
+
+	// write the pack file
+	packFile, err = yaml.Marshal(packData)
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(packPath, packFile, 0644)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Generate a query pack containing the given query file.
+func generateQueryPack(queryFile string) (string, error) {
+	fmt.Printf("Generating query pack for %s\n", queryFile)
+
+	// create a temporary directory to hold the query pack
+	queryPackDir, err := ioutil.TempDir("", "query-pack-")
+	if err != nil {
+		log.Fatal(err)
+	}
+	// TODO: uncomment this line when we're done debugging
+	//defer os.RemoveAll(queryPackDir)
+
+	queryFile, err = filepath.Abs(queryFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+	if _, err := os.Stat(queryFile); errors.Is(err, os.ErrNotExist) {
+		log.Fatal(fmt.Sprintf("Query file %s does not exist", queryFile))
+		os.Exit(1)
+	}
+	originalPackRoot := findPackRoot(queryFile)
+	packRelativePath, _ := filepath.Rel(originalPackRoot, queryFile)
+	targetQueryFileName := filepath.Join(queryPackDir, packRelativePath)
+
+	if _, err := os.Stat(filepath.Join(originalPackRoot, "qlpack.yml")); errors.Is(err, os.ErrNotExist) {
+		// qlpack.yml not found, generate a synthetic one
+		fmt.Printf("QLPack does not exist. Generating synthetic one for %s\n", queryFile)
+		// copy only the query file to the query pack directory
+		err := copyFile(queryFile, targetQueryFileName)
+		if err != nil {
+			log.Fatal(err)
+		}
+		// generate a synthetic qlpack.yml
+		td := struct {
+			Language string
+			Name     string
+			Query    string
+		}{
+			Language: language,
+			Name:     "codeql-remote/query",
+			Query:    strings.Replace(packRelativePath, string(os.PathSeparator), "/", -1),
+		}
+		t, err := template.New("").Parse(`name: {{ .Name }}
+version: 0.0.0
+dependencies:
+  codeql/{{ .Language }}-all: "*"
+defaultSuite:
+  description: Query suite for variant analysis
+  query: {{ .Query }}`)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		f, err := os.Create(filepath.Join(queryPackDir, "qlpack.yml"))
+		defer f.Close()
+		if err != nil {
+			log.Fatal(err)
+		}
+		err = t.Execute(f, td)
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Printf("Copied QLPack files to %s\n", queryPackDir)
+	} else {
+		// don't include all query files in the QLPacks. We only want the queryFile to be copied.
+		fmt.Printf("QLPack exists, stripping all other queries from %s\n", originalPackRoot)
+		toCopy := packPacklist(originalPackRoot, false)
+		// also copy the lock file (either new name or old name) and the query file itself (these are not included in the packlist)
+		lockFileNew := filepath.Join(originalPackRoot, "qlpack.lock.yml")
+		lockFileOld := filepath.Join(originalPackRoot, "codeql-pack.lock.yml")
+		candidateFiles := []string{lockFileNew, lockFileOld, queryFile}
+		for _, candidateFile := range candidateFiles {
+			if _, err := os.Stat(candidateFile); !errors.Is(err, os.ErrNotExist) {
+				// if the file exists, copy it
+				toCopy = append(toCopy, candidateFile)
+			}
+		}
+		// copy the files to the queryPackDir directory
+		fmt.Printf("Preparing stripped QLPack in %s\n", queryPackDir)
+		for _, srcPath := range toCopy {
+			relPath, _ := filepath.Rel(originalPackRoot, srcPath)
+			targetPath := filepath.Join(queryPackDir, relPath)
+			//fmt.Printf("Copying %s to %s\n", srcPath, targetPath)
+			err := copyFile(srcPath, targetPath)
+			if err != nil {
+				log.Fatal(err)
+			}
+		}
+		fmt.Printf("Fixing QLPack in %s\n", queryPackDir)
+		fixPackFile(queryPackDir, packRelativePath)
+	}
+
+	// assuming we are using 2.11.3 or later so Qlx remote is supported
+	ccache := filepath.Join(originalPackRoot, ".cache")
+	precompilationOpts := []string{"--qlx", "--no-default-compilation-cache", "--compilation-cache=" + ccache}
+	bundlePath := filepath.Join(filepath.Dir(queryPackDir), fmt.Sprintf("qlpack-%s-generated.tgz", uuid.New().String()))
+
+	// install the pack dependencies
+	fmt.Print("Installing QLPack dependencies\n")
+	args := []string{"pack", "install", queryPackDir}
+	stdouterr, err := exec.Command("codeql", args...).CombinedOutput()
+	if err != nil {
+		fmt.Printf("`codeql pack bundle` failed with error: %v\n", string(stdouterr))
+		return "", fmt.Errorf("Failed to install query pack: %v", err)
+	}
+	// bundle the query pack
+	fmt.Print("Compiling and bundling the QLPack (This may take a while)\n")
+	args = []string{"pack", "bundle", "-o", bundlePath, queryPackDir}
+	args = append(args, precompilationOpts...)
+	stdouterr, err = exec.Command("codeql", args...).CombinedOutput()
+	if err != nil {
+		fmt.Printf("`codeql pack bundle` failed with error: %v\n", string(stdouterr))
+		return "", fmt.Errorf("Failed to bundle query pack: %v\n", err)
+	}
+
+	// open the bundle file and encode it as base64
+	bundleFile, err := os.Open(bundlePath)
+	if err != nil {
+		return "", fmt.Errorf("Failed to open bundle file: %v\n", err)
+	}
+	defer bundleFile.Close()
+	bundleBytes, err := ioutil.ReadAll(bundleFile)
+	if err != nil {
+		return "", fmt.Errorf("Failed to read bundle file: %v\n", err)
+	}
+	bundleBase64 := base64.StdEncoding.EncodeToString(bundleBytes)
+
+	return bundleBase64, nil
+}
+
+// Requests a query to be run against `respositories` on the given `controller`.
+func submitRun(repoChunk []string, bundle string) (int, error) {
+	// See https://github.com/github/github/blob/master/app/api/description/operations/code-scanning/create-variant-analysis.yaml
+	opts := api.ClientOptions{
+		Headers: map[string]string{"Accept": "application/vnd.github.v3+json"},
+	}
+	client, err := gh.RESTClient(&opts)
+	if err != nil {
+		return -1, err
+	}
+	body := struct {
+		Repositories []string `json:"repositories"`
+		Language     string   `json:"language"`
+		Pack         string   `json:"query_pack"`
+		Ref          string   `json:"action_repo_ref"`
+	}{
+		Repositories: repoChunk,
+		Language:     language,
+		Pack:         bundle,
+		Ref:          "main",
+	}
+	var buf bytes.Buffer
+	err = json.NewEncoder(&buf).Encode(body)
+	if err != nil {
+		return -1, err
+	}
+	response := make(map[string]interface{})
+	err = client.Post(fmt.Sprintf("repos/%s/code-scanning/codeql/variant-analyses", controller), &buf, &response)
+	if err != nil {
+		return -1, err
+	}
+	id := int(response["id"].(float64))
+	return id, nil
+}
+
+func getRunDetails(runId int) (map[string]interface{}, error) {
+	// See https://github.com/github/github/blob/master/app/api/description/operations/code-scanning/get-variant-analysis.yaml
+	opts := api.ClientOptions{
+		Headers: map[string]string{"Accept": "application/vnd.github.v3+json"},
+	}
+	client, err := gh.RESTClient(&opts)
+	if err != nil {
+		return nil, err
+	}
+	response := make(map[string]interface{})
+	err = client.Get(fmt.Sprintf("repos/%s/code-scanning/codeql/variant-analyses/%d", controller, runId), &response)
+	if err != nil {
+		return nil, err
+	}
+	return response, nil
+}
+
+func getRunRepositoryDetails(runId int, nwo string) (map[string]interface{}, error) {
+	// See https://github.com/github/github/blob/master/app/api/description/operations/code-scanning/get-variant-analysis-repo-task.yaml
+	opts := api.ClientOptions{
+		Headers: map[string]string{"Accept": "application/vnd.github.v3+json"},
+	}
+	client, err := gh.RESTClient(&opts)
+	if err != nil {
+		return nil, err
+	}
+	response := make(map[string]interface{})
+	err = client.Get(fmt.Sprintf("repos/%s/code-scanning/codeql/variant-analyses/%d/repos/%s", controller, runId, nwo), &response)
+	if err != nil {
+		return nil, err
+	}
+	return response, nil
+}
+
+func downloadArtifact(url string, outputDir string) (string, error) {
+	client, err := gh.HTTPClient(nil)
+	if err != nil {
+		return "", err
+	}
+	resp, err := client.Get(url)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, zf := range zipReader.File {
+		if zf.Name != "results.sarif" && zf.Name != "results.bqrs" {
+			continue
+		}
+		f, err := zf.Open()
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer f.Close()
+		bytes, err := ioutil.ReadAll(f)
+		if err != nil {
+			log.Fatal(err)
+		}
+		resultPath := ""
+		if zf.Name == "results.bqrs" {
+			resultPath = filepath.Join(outputDir, "FOO.bqrs")
+		} else if zf.Name == "results.sarif" {
+			resultPath = filepath.Join(outputDir, "FOO.sarif")
+		}
+		if resultPath != "" {
+			err = ioutil.WriteFile(resultPath, bytes, os.ModePerm)
+			if err != nil {
+				return "", err
+			}
+			return resultPath, nil
+		}
+	}
+	return "", errors.New("No results.sarif file found in artifact")
+}
+
+func downloadDatabase(nwo string, lang string, targetPath string) error {
+	opts := api.ClientOptions{
+		Headers: map[string]string{"Accept": "application/zip"},
+	}
+	client, err := gh.HTTPClient(&opts)
+	if err != nil {
+		return err
+	}
+	resp, err := client.Get(fmt.Sprintf("https://api.github.com/repos/%s/code-scanning/codeql/databases/%s", nwo, lang))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	bytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(targetPath, bytes, os.ModePerm)
+	return nil
+}
+
+func saveInCache(name string, ids []int) error {
+	configData, err := getConfig(configFilePath)
+	if err != nil {
+		return err
+	}
+	cache := configData.Cache
+	if cache == nil {
+		cache = map[string][]int{}
+	}
+	if cache[name] == nil {
+		cache[name] = ids
+	} else {
+		cache[name] = append(cache[name], ids...)
+	}
+	// marshal config data to yaml
+	configDataYaml, err := yaml.Marshal(configData)
+	if err != nil {
+		return err
+	}
+	// write config data to file
+	err = ioutil.WriteFile(configFilePath, configDataYaml, os.ModePerm)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func loadFromCache(name string) ([]int, error) {
+	configData, err := getConfig(configFilePath)
+	if err != nil {
+		return nil, err
+	}
+	if configData.Cache != nil {
+		if configData.Cache[name] != nil {
+			return configData.Cache[name], nil
+		}
+	}
+	return []int{}, nil
+}
+
+func getConfig(path string) (Config, error) {
+	configFile, err := ioutil.ReadFile(configFilePath)
+	var configData Config
+	if err != nil {
+		return configData, err
+	}
+	err = yaml.Unmarshal(configFile, &configData)
+	if err != nil {
+		log.Fatal(err)
+	}
+	return configData, nil
+}
+
+type Config struct {
+	Controller string           `yaml:"controller"`
+	ListFile   string           `yaml:"listFile"`
+	Cache      map[string][]int `yaml:"cache"`
+}
+
+func main() {
+
+	// read config file
+	homePath := os.Getenv("XDG_CONFIG_HOME")
+	if homePath == "" {
+		homePath = os.Getenv("HOME")
+	}
+	configFilePath = filepath.Join(homePath, ".config", "mrva", "config.yml")
+	if _, err := os.Stat(configFilePath); os.IsNotExist(err) {
+		// create config file if it doesn't exist
+		// since we will use it for the name/ids cache
+		err := os.MkdirAll(filepath.Dir(configFilePath), os.ModePerm)
+		if err != nil {
+			log.Println("Failed to create config file directory")
+		}
+		// create empty file at configFilePath
+		configFile, err := os.Create(configFilePath)
+		if err != nil {
+			log.Fatal(err, "Failed to create config file")
+		}
+		configFile.Close()
+	}
+	configData, err := getConfig(configFilePath)
+	if err != nil {
+		log.Fatal(err)
+	}
+	if configData.Controller != "" {
+		controller = configData.Controller
+	}
+	if configData.ListFile != "" {
+		listFile = configData.ListFile
+	}
+
+	flag.Parse()
+	args := flag.Args()
+	if len(args) == 0 {
+		log.Fatal("Please specify a subcommand.")
+	}
+	cmd, args := args[0], args[1:]
+
+	switch cmd {
+	case "submit":
+		submit(args)
+	case "download":
+		download(args)
+	default:
+		log.Fatalf("Unrecognized command %q. "+
+			"Command must be one of: submit, download", cmd)
+	}
+}
+
+func submit(args []string) {
+	flag := flag.NewFlagSet("mrva submit", flag.ExitOnError)
+	queryFileFlag := flag.String("query", "", "Path to query file")
+	controllerFlag := flag.String("controller", "", "MRVA controller repository (overrides config file)")
+	listFileFlag := flag.String("list-file", "", "Path to repo list file (overrides config file)")
+	listFlag := flag.String("list", "", "Name of repo list")
+	langFlag := flag.String("lang", "", "DB language")
+	nameFlag := flag.String("name", "", "Name of run (optional)")
+
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, `gh mrva - submit and download CodeQL queries from MRVA
+Usage:
+	gh mrva submit --controller <controller> --lang <language> [--name <run name>] --list-file <list file> --list <list> --query <query>
+
+Flags:
+`)
+		flag.PrintDefaults()
+	}
+	flag.Parse(args)
+
+	// set global variables
+	if *langFlag != "" {
+		language = *langFlag
+	}
+	if *nameFlag != "" {
+		runName = *nameFlag
+	}
+	if *controllerFlag != "" {
+		controller = *controllerFlag
+	}
+	if *listFileFlag != "" {
+		listFile = *listFileFlag
+	}
+
+	if controller == "" || language == "" || listFile == "" || *listFlag == "" || *queryFileFlag == "" {
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	repositories, err := resolveRepositories(listFile, *listFlag)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("Requesting run for %d repositories from %s:%s\n", len(repositories), listFile, *listFlag)
+
+	encodedBundle, err := generateQueryPack(*queryFileFlag)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println("Generated encoded bundle")
+
+	var chunks [][]string
+	for i := 0; i < len(repositories); i += MAX_MRVA_REPOSITORIES {
+		end := i + MAX_MRVA_REPOSITORIES
+		if end > len(repositories) {
+			end = len(repositories)
+		}
+		chunks = append(chunks, repositories[i:end])
+	}
+	var ids []int
+	for _, chunk := range chunks {
+		id, err := submitRun(chunk, encodedBundle)
+		if err != nil {
+			log.Fatal(err)
+		}
+		ids = append(ids, id)
+	}
+	fmt.Printf("Submitted run %v\n", ids)
+	if runName != "" {
+		err = saveInCache(runName, ids)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+}
+
+func download(args []string) {
+	flag := flag.NewFlagSet("mrva submit", flag.ExitOnError)
+	runFlag := flag.Int("run", 0, "MRVA run ID")
+	outputDirFlag := flag.String("output-dir", "", "Output directory")
+	downloadDBsFlag := flag.Bool("download-dbs", false, "Download databases (optional)")
+	controllerFlag := flag.String("controller", "", "MRVA controller repository (overrides config file)")
+	langFlag := flag.String("lang", "", "DB language")
+	nameFlag := flag.String("name", "", "Name of run (optional)")
+
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, `gh mrva - submit and download CodeQL queries from MRVA
+Usage:
+	gh mrva download --run <run id> --lang <language> --controller <controller> --output-dir <output directory> [--name <run name>] [--download-dbs]
+
+Flags:
+`)
+		flag.PrintDefaults()
+	}
+
+	flag.Parse(args)
+
+	// set global variables
+	if *langFlag != "" {
+		language = *langFlag
+	}
+	if *nameFlag != "" {
+		runName = *nameFlag
+	}
+	if *controllerFlag != "" {
+		controller = *controllerFlag
+	}
+
+	if controller == "" || language == "" || (*runFlag == 0 && runName == "") || *outputDirFlag == "" {
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	// if outputDirFlag does not exist, create it
+	if _, err := os.Stat(*outputDirFlag); os.IsNotExist(err) {
+		err := os.MkdirAll(*outputDirFlag, os.ModePerm)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	runIds := []int{}
+	if *runFlag > 0 {
+		runIds = []int{*runFlag}
+	} else if runName != "" {
+		ids, err := loadFromCache(runName)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if len(ids) > 0 {
+			runIds = ids
+		}
+	}
+
+	for _, runId := range runIds {
+		fmt.Printf("Downloading MRVA results for %s (%d)\n", controller, runId)
+		// check if the run is complete
+		runDetails, err := getRunDetails(runId)
+		fmt.Printf("Status: %v\n", runDetails["status"])
+		if err != nil {
+			log.Fatal(err)
+		}
+		if runDetails["status"] == "in_progress" {
+			log.Printf("Run %d is not complete yet. Please try again later.", runId)
+			return
+		}
+		for _, r := range runDetails["scanned_repositories"].([]interface{}) {
+			repo := r.(map[string]interface{})
+			result_count := repo["result_count"]
+			repoInfo := repo["repository"].(map[string]interface{})
+			nwo := repoInfo["full_name"].(string)
+			if result_count != nil && result_count.(float64) > 0 {
+				fmt.Printf("Repo %s has %d results\n", nwo, int(result_count.(float64)))
+				resultPath := filepath.Join(*outputDirFlag, fmt.Sprintf("%s.sarif", strings.Replace(nwo, "/", "_", -1)))
+
+				if _, err := os.Stat(resultPath); errors.Is(err, os.ErrNotExist) {
+
+					// download artifact (BQRS or SARIF)
+					fmt.Printf("Downloading results for %s\n", repoInfo["full_name"])
+					runRepositoryDetails, err := getRunRepositoryDetails(runId, nwo)
+					if err != nil {
+						log.Fatal(err)
+					}
+					// download the results
+					artifactPath, err := downloadArtifact(runRepositoryDetails["artifact_url"].(string), *outputDirFlag)
+					if err != nil {
+						log.Fatal(err)
+					}
+					fmt.Printf("Artifact path: %s\n", artifactPath)
+				}
+				if *downloadDBsFlag {
+					// download database
+					targetPath := filepath.Join(*outputDirFlag, fmt.Sprintf("%s_%s_db.zip", strings.Replace(nwo, "/", "_", -1), language))
+					if _, err := os.Stat(targetPath); errors.Is(err, os.ErrNotExist) {
+						fmt.Printf("Downloading database for %s\n", nwo)
+						err = downloadDatabase(nwo, language, targetPath)
+						if err != nil {
+							log.Fatal(err)
+						}
+						fmt.Printf("Database path: %s\n", targetPath)
+					}
+				}
+			}
+		}
+	}
+}

tests/.cache/data/15/a4/15a4cf726148fed00e9291fa46dd48edf9507125

@@ -0,0 +1,20 @@
+ #      Precompiled CodeQL query
+---
+format:
+- 202210190
+creator: "2.12.4"
+name: "query.ql"
+dbscheme: "934bf10b4bd34cf648893efcd1d0d7be9471d39f"
+stages:
+- cached: "d366cddf0b9c37dfe2b2b40f6b96537096e08d00"
+  debugInfo: "fecaa121855b37cb5060daceb1adc2e02056f801"
+- cached: "248f1f782c7cbe975da92bc515b37371b1c1136b"
+  debugInfo: "fecaa121855b37cb5060daceb1adc2e02056f801"
+- cached: "cffbc7d7aa728a15b1e164d22564f43a1d232b17"
+  debugInfo: "fecaa121855b37cb5060daceb1adc2e02056f801"
+- cached: "4d40c42c794541aa678d01096f984031c66b3e68"
+  debugInfo: "fecaa121855b37cb5060daceb1adc2e02056f801"
+results:
+  '#select':
+    resultArranger: "1\"ma\"ei12"
+inputsDigest: "15a4cf726148fed00e9291fa46dd48edf9507125"

tests/.cache/data/b2/e5/b2e5a6727f6474d5add4e280b09df9b3eb2a7a02

@@ -0,0 +1,20 @@
+ #      Precompiled CodeQL query
+---
+format:
+- 202210190
+creator: "2.12.4"
+name: "query.ql"
+dbscheme: "934bf10b4bd34cf648893efcd1d0d7be9471d39f"
+stages:
+- cached: "d366cddf0b9c37dfe2b2b40f6b96537096e08d00"
+  debugInfo: "fecaa121855b37cb5060daceb1adc2e02056f801"
+- cached: "248f1f782c7cbe975da92bc515b37371b1c1136b"
+  debugInfo: "fecaa121855b37cb5060daceb1adc2e02056f801"
+- cached: "cffbc7d7aa728a15b1e164d22564f43a1d232b17"
+  debugInfo: "fecaa121855b37cb5060daceb1adc2e02056f801"
+- cached: "4d40c42c794541aa678d01096f984031c66b3e68"
+  debugInfo: "fecaa121855b37cb5060daceb1adc2e02056f801"
+results:
+  '#select':
+    resultArranger: "1\"ma\"ei12"
+inputsDigest: "b2e5a6727f6474d5add4e280b09df9b3eb2a7a02"

tests/query.ql

@@ -0,0 +1,4 @@
+import java
+from MethodAccess ma
+where ma.getMethod().getName().matches("a%")
+select ma

tests/repos.json

@@ -0,0 +1,3 @@
+{
+  "sample": ["apache/logging-log4j2"]
+}