diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..dfda64d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,14 @@ +name: release +on: + push: + tags: + - "v*" +permissions: + contents: write + +jobs: + release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: cli/gh-extension-precompile@v1 diff --git a/README.md b/README.md index ae4f339..417182c 100644 --- a/README.md +++ b/README.md @@ -1 +1,29 @@ -# gh-mrva \ No newline at end of file +# gh-mrva + +## Configuration + +A configuration file will be created in `~/.config/mrva/config.yml`. The following options are supported: +- `controller`: NWO of the MRVA controller to use +- `listFile`: Path to the JSON file containing the target repos + +## Usage + +Until the extension gets published you can use `go run .` instead of `gh mrva` + +### Submit a new query + +```bash +gh mrva submit [--controller ] --lang [--list-file ] --list --query [--name ] +``` + +Note: `controller` and `list-file` are only optionals if defined in the configuration file +Note: if a `name` (any arbitrary name) is provided, the resulting run IDs will be stored in the configuration file so they can be referenced later for download + +### Download the results + +```bash +gh mrva download [--controller ] --lang --output-dir [--name | --run ] [--download-dbs] +``` + +Note: `controller` is only optionals if defined in the configuration file +Note: if a `name` is provided, the run ID is not necessary and instead `gh-mrva` will download the artifacts associated that `name` as found in the configuration file diff --git a/gh-mrva b/gh-mrva new file mode 100755 index 0000000..efc797c Binary files /dev/null and b/gh-mrva differ diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..dcf4dd6 --- /dev/null +++ b/go.mod @@ -0,0 +1,26 @@ +module github.com/pwntester/gh-mrva + +go 1.19 + +require github.com/cli/go-gh v1.2.1 + +require ( + github.com/cli/safeexec v1.0.0 // indirect + github.com/cli/shurcooL-graphql v0.0.2 // indirect + github.com/google/uuid v1.3.0 // direct + github.com/henvic/httpretty v0.0.6 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/mattn/go-isatty v0.0.16 // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/muesli/termenv v0.12.0 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/thlib/go-timezone-local v0.0.0-20210907160436-ef149e42d28e // indirect + github.com/tidwall/gjson v1.14.4 // direct + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/sys v0.5.0 // indirect + golang.org/x/term v0.5.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // direct +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..55e1eea --- /dev/null +++ b/go.sum @@ -0,0 +1,71 @@ +github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= +github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= +github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/cli/go-gh v1.2.1 h1:xFrjejSsgPiwXFP6VYynKWwxLQcNJy3Twbu82ZDlR/o= +github.com/cli/go-gh v1.2.1/go.mod h1:Jxk8X+TCO4Ui/GarwY9tByWm/8zp4jJktzVZNlTW5VM= +github.com/cli/safeexec v1.0.0 h1:0VngyaIyqACHdcMNWfo6+KdUYnqEr2Sg+bSP1pdF+dI= +github.com/cli/safeexec v1.0.0/go.mod h1:Z/D4tTN8Vs5gXYHDCbaM1S/anmEDnJb1iW0+EJ5zx3Q= +github.com/cli/shurcooL-graphql v0.0.2 h1:rwP5/qQQ2fM0TzkUTwtt6E2LbIYf6R+39cUXTa04NYk= +github.com/cli/shurcooL-graphql v0.0.2/go.mod h1:tlrLmw/n5Q/+4qSvosT+9/W5zc8ZMjnJeYBxSdb4nWA= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= +github.com/henvic/httpretty v0.0.6 h1:JdzGzKZBajBfnvlMALXXMVQWxWMF/ofTy8C3/OSUTxs= +github.com/henvic/httpretty v0.0.6/go.mod h1:X38wLjWXHkXT7r2+uK8LjCMne9rsuNaBLJ+5cU2/Pmo= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= +github.com/muesli/termenv v0.12.0 h1:KuQRUE3PgxRFWhq4gHvZtPSLCGDqM5q/cYr1pZ39ytc= +github.com/muesli/termenv v0.12.0/go.mod h1:WCCv32tusQ/EEZ5S8oUIIrC/nIuBcxCVqlN4Xfkv+7A= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/thlib/go-timezone-local v0.0.0-20210907160436-ef149e42d28e h1:BuzhfgfWQbX0dWzYzT1zsORLnHRv3bcRcsaUk0VmXA8= +github.com/thlib/go-timezone-local v0.0.0-20210907160436-ef149e42d28e/go.mod h1:/Tnicc6m/lsJE0irFMA0LfIwTBo4QP7A8IfyIv4zZKI= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +golang.org/x/net v0.0.0-20220923203811-8be639271d50/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..95d0f44 --- /dev/null +++ b/main.go @@ -0,0 +1,739 @@ +package main + +import ( + "archive/zip" + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "flag" + "fmt" + "github.com/cli/go-gh" + "github.com/cli/go-gh/pkg/api" + "github.com/google/uuid" + "github.com/tidwall/gjson" + "gopkg.in/yaml.v3" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "text/template" +) + +const ( + MAX_MRVA_REPOSITORIES = 1000 +) + +var ( + configFilePath = "" + controller = "" + language = "" + runName = "" + listFile = "" +) + +func registerGlobalFlags(fset *flag.FlagSet) { + // parsing subcommands in GoLang: https://abhinavg.net/2022/08/13/flag-subcommand/ + flag.VisitAll(func(f *flag.Flag) { + fset.Var(f.Value, f.Name, f.Usage) + }) +} + +func resolveRepositories(listFile string, list string) ([]string, error) { + fmt.Printf("Resolving %s repositories from %s\n", list, listFile) + jsonFile, err := os.Open(listFile) + if err != nil { + fmt.Println(err) + return nil, err + } + defer jsonFile.Close() + byteValue, _ := ioutil.ReadAll(jsonFile) + json := string(byteValue) + repositories := gjson.Get(json, list) + var result []string + for _, x := range repositories.Array() { + result = append(result, x.String()) + } + return result, nil +} + +func findPackRoot(queryFile string) string { + // Starting on the directory of queryPackDir, go down until a qlpack.yml find is found. return that directory + // If no qlpack.yml is found, return the directory of queryFile + currentDir := filepath.Dir(queryFile) + for currentDir != "/" { + if _, err := os.Stat(filepath.Join(currentDir, "qlpack.yml")); errors.Is(err, os.ErrNotExist) { + // qlpack.yml not found, go up one level + currentDir = filepath.Dir(currentDir) + } else { + return currentDir + } + } + return filepath.Dir(queryFile) +} + +func packPacklist(dir string, includeQueries bool) []string { + // since 2.7.1, packlist returns an object with a "paths" property that is a list of packs. + args := []string{"pack", "packlist", "--format=json"} + if !includeQueries { + args = append(args, "--no-include-queries") + } + args = append(args, dir) + json, err := exec.Command("codeql", args...).Output() + // parse the json + packlist := gjson.Get(string(json), "paths") + if err != nil { + log.Fatal(err) + } + var result []string + for _, x := range packlist.Array() { + result = append(result, x.String()) + } + return result +} + +func copyFile(srcPath string, targetPath string) error { + err := os.MkdirAll(filepath.Dir(targetPath), os.ModePerm) + if err != nil { + return err + } + bytesRead, err := ioutil.ReadFile(srcPath) + if err != nil { + return err + } + err = ioutil.WriteFile(targetPath, bytesRead, 0644) + if err != nil { + return err + } + return nil +} + +// Fixes the qlpack.yml file to be correct in the context of the MRVA request. +// Performs the following fixes: +// - Updates the default suite of the query pack. This is used to ensure +// only the specified query is run. +// - Ensures the query pack name is set to the name expected by the server. +// - Removes any `${workspace}` version references from the qlpack.yml file. Converts them +// to `*` versions. +// @param queryPackDir The directory containing the query pack +// @param packRelativePath The relative path to the query pack from the root of the query pack +func fixPackFile(queryPackDir string, packRelativePath string) error { + packPath := filepath.Join(queryPackDir, "qlpack.yml") + packFile, err := ioutil.ReadFile(packPath) + if err != nil { + return err + } + var packData map[string]interface{} + err = yaml.Unmarshal(packFile, &packData) + if err != nil { + return err + } + // update the default suite + defaultSuiteFile := packData["defaultSuiteFile"] + if defaultSuiteFile != nil { + // remove the defaultSuiteFile property + delete(packData, "defaultSuiteFile") + } + packData["defaultSuite"] = map[string]string{ + "query": packRelativePath, + "description": "Query suite for Variant Analysis", + } + + // update the name + packData["name"] = "codeql-remote/query" + + // remove any `${workspace}` version references + dependencies := packData["dependencies"] + if dependencies != nil { + // for key and value in dependencies + for key, value := range dependencies.(map[string]interface{}) { + // if value is a string and value contains `${workspace}` + if value == "${workspace}" { + // replace the value with `*` + packData["dependencies"].(map[string]string)[key] = "*" + } + } + } + + // write the pack file + packFile, err = yaml.Marshal(packData) + if err != nil { + return err + } + err = ioutil.WriteFile(packPath, packFile, 0644) + if err != nil { + return err + } + return nil +} + +// Generate a query pack containing the given query file. +func generateQueryPack(queryFile string) (string, error) { + fmt.Printf("Generating query pack for %s\n", queryFile) + + // create a temporary directory to hold the query pack + queryPackDir, err := ioutil.TempDir("", "query-pack-") + if err != nil { + log.Fatal(err) + } + // TODO: uncomment this line when we're done debugging + //defer os.RemoveAll(queryPackDir) + + queryFile, err = filepath.Abs(queryFile) + if err != nil { + log.Fatal(err) + } + if _, err := os.Stat(queryFile); errors.Is(err, os.ErrNotExist) { + log.Fatal(fmt.Sprintf("Query file %s does not exist", queryFile)) + os.Exit(1) + } + originalPackRoot := findPackRoot(queryFile) + packRelativePath, _ := filepath.Rel(originalPackRoot, queryFile) + targetQueryFileName := filepath.Join(queryPackDir, packRelativePath) + + if _, err := os.Stat(filepath.Join(originalPackRoot, "qlpack.yml")); errors.Is(err, os.ErrNotExist) { + // qlpack.yml not found, generate a synthetic one + fmt.Printf("QLPack does not exist. Generating synthetic one for %s\n", queryFile) + // copy only the query file to the query pack directory + err := copyFile(queryFile, targetQueryFileName) + if err != nil { + log.Fatal(err) + } + // generate a synthetic qlpack.yml + td := struct { + Language string + Name string + Query string + }{ + Language: language, + Name: "codeql-remote/query", + Query: strings.Replace(packRelativePath, string(os.PathSeparator), "/", -1), + } + t, err := template.New("").Parse(`name: {{ .Name }} +version: 0.0.0 +dependencies: + codeql/{{ .Language }}-all: "*" +defaultSuite: + description: Query suite for variant analysis + query: {{ .Query }}`) + if err != nil { + log.Fatal(err) + } + + f, err := os.Create(filepath.Join(queryPackDir, "qlpack.yml")) + defer f.Close() + if err != nil { + log.Fatal(err) + } + err = t.Execute(f, td) + if err != nil { + log.Fatal(err) + } + fmt.Printf("Copied QLPack files to %s\n", queryPackDir) + } else { + // don't include all query files in the QLPacks. We only want the queryFile to be copied. + fmt.Printf("QLPack exists, stripping all other queries from %s\n", originalPackRoot) + toCopy := packPacklist(originalPackRoot, false) + // also copy the lock file (either new name or old name) and the query file itself (these are not included in the packlist) + lockFileNew := filepath.Join(originalPackRoot, "qlpack.lock.yml") + lockFileOld := filepath.Join(originalPackRoot, "codeql-pack.lock.yml") + candidateFiles := []string{lockFileNew, lockFileOld, queryFile} + for _, candidateFile := range candidateFiles { + if _, err := os.Stat(candidateFile); !errors.Is(err, os.ErrNotExist) { + // if the file exists, copy it + toCopy = append(toCopy, candidateFile) + } + } + // copy the files to the queryPackDir directory + fmt.Printf("Preparing stripped QLPack in %s\n", queryPackDir) + for _, srcPath := range toCopy { + relPath, _ := filepath.Rel(originalPackRoot, srcPath) + targetPath := filepath.Join(queryPackDir, relPath) + //fmt.Printf("Copying %s to %s\n", srcPath, targetPath) + err := copyFile(srcPath, targetPath) + if err != nil { + log.Fatal(err) + } + } + fmt.Printf("Fixing QLPack in %s\n", queryPackDir) + fixPackFile(queryPackDir, packRelativePath) + } + + // assuming we are using 2.11.3 or later so Qlx remote is supported + ccache := filepath.Join(originalPackRoot, ".cache") + precompilationOpts := []string{"--qlx", "--no-default-compilation-cache", "--compilation-cache=" + ccache} + bundlePath := filepath.Join(filepath.Dir(queryPackDir), fmt.Sprintf("qlpack-%s-generated.tgz", uuid.New().String())) + + // install the pack dependencies + fmt.Print("Installing QLPack dependencies\n") + args := []string{"pack", "install", queryPackDir} + stdouterr, err := exec.Command("codeql", args...).CombinedOutput() + if err != nil { + fmt.Printf("`codeql pack bundle` failed with error: %v\n", string(stdouterr)) + return "", fmt.Errorf("Failed to install query pack: %v", err) + } + // bundle the query pack + fmt.Print("Compiling and bundling the QLPack (This may take a while)\n") + args = []string{"pack", "bundle", "-o", bundlePath, queryPackDir} + args = append(args, precompilationOpts...) + stdouterr, err = exec.Command("codeql", args...).CombinedOutput() + if err != nil { + fmt.Printf("`codeql pack bundle` failed with error: %v\n", string(stdouterr)) + return "", fmt.Errorf("Failed to bundle query pack: %v\n", err) + } + + // open the bundle file and encode it as base64 + bundleFile, err := os.Open(bundlePath) + if err != nil { + return "", fmt.Errorf("Failed to open bundle file: %v\n", err) + } + defer bundleFile.Close() + bundleBytes, err := ioutil.ReadAll(bundleFile) + if err != nil { + return "", fmt.Errorf("Failed to read bundle file: %v\n", err) + } + bundleBase64 := base64.StdEncoding.EncodeToString(bundleBytes) + + return bundleBase64, nil +} + +// Requests a query to be run against `respositories` on the given `controller`. +func submitRun(repoChunk []string, bundle string) (int, error) { + // See https://github.com/github/github/blob/master/app/api/description/operations/code-scanning/create-variant-analysis.yaml + opts := api.ClientOptions{ + Headers: map[string]string{"Accept": "application/vnd.github.v3+json"}, + } + client, err := gh.RESTClient(&opts) + if err != nil { + return -1, err + } + body := struct { + Repositories []string `json:"repositories"` + Language string `json:"language"` + Pack string `json:"query_pack"` + Ref string `json:"action_repo_ref"` + }{ + Repositories: repoChunk, + Language: language, + Pack: bundle, + Ref: "main", + } + var buf bytes.Buffer + err = json.NewEncoder(&buf).Encode(body) + if err != nil { + return -1, err + } + response := make(map[string]interface{}) + err = client.Post(fmt.Sprintf("repos/%s/code-scanning/codeql/variant-analyses", controller), &buf, &response) + if err != nil { + return -1, err + } + id := int(response["id"].(float64)) + return id, nil +} + +func getRunDetails(runId int) (map[string]interface{}, error) { + // See https://github.com/github/github/blob/master/app/api/description/operations/code-scanning/get-variant-analysis.yaml + opts := api.ClientOptions{ + Headers: map[string]string{"Accept": "application/vnd.github.v3+json"}, + } + client, err := gh.RESTClient(&opts) + if err != nil { + return nil, err + } + response := make(map[string]interface{}) + err = client.Get(fmt.Sprintf("repos/%s/code-scanning/codeql/variant-analyses/%d", controller, runId), &response) + if err != nil { + return nil, err + } + return response, nil +} + +func getRunRepositoryDetails(runId int, nwo string) (map[string]interface{}, error) { + // See https://github.com/github/github/blob/master/app/api/description/operations/code-scanning/get-variant-analysis-repo-task.yaml + opts := api.ClientOptions{ + Headers: map[string]string{"Accept": "application/vnd.github.v3+json"}, + } + client, err := gh.RESTClient(&opts) + if err != nil { + return nil, err + } + response := make(map[string]interface{}) + err = client.Get(fmt.Sprintf("repos/%s/code-scanning/codeql/variant-analyses/%d/repos/%s", controller, runId, nwo), &response) + if err != nil { + return nil, err + } + return response, nil +} + +func downloadArtifact(url string, outputDir string) (string, error) { + client, err := gh.HTTPClient(nil) + if err != nil { + return "", err + } + resp, err := client.Get(url) + if err != nil { + return "", err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Fatal(err) + } + + zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) + if err != nil { + log.Fatal(err) + } + + for _, zf := range zipReader.File { + if zf.Name != "results.sarif" && zf.Name != "results.bqrs" { + continue + } + f, err := zf.Open() + if err != nil { + log.Fatal(err) + } + defer f.Close() + bytes, err := ioutil.ReadAll(f) + if err != nil { + log.Fatal(err) + } + resultPath := "" + if zf.Name == "results.bqrs" { + resultPath = filepath.Join(outputDir, "FOO.bqrs") + } else if zf.Name == "results.sarif" { + resultPath = filepath.Join(outputDir, "FOO.sarif") + } + if resultPath != "" { + err = ioutil.WriteFile(resultPath, bytes, os.ModePerm) + if err != nil { + return "", err + } + return resultPath, nil + } + } + return "", errors.New("No results.sarif file found in artifact") +} + +func downloadDatabase(nwo string, lang string, targetPath string) error { + opts := api.ClientOptions{ + Headers: map[string]string{"Accept": "application/zip"}, + } + client, err := gh.HTTPClient(&opts) + if err != nil { + return err + } + resp, err := client.Get(fmt.Sprintf("https://api.github.com/repos/%s/code-scanning/codeql/databases/%s", nwo, lang)) + if err != nil { + return err + } + defer resp.Body.Close() + + bytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + err = ioutil.WriteFile(targetPath, bytes, os.ModePerm) + return nil +} + +func saveInCache(name string, ids []int) error { + configData, err := getConfig(configFilePath) + if err != nil { + return err + } + cache := configData.Cache + if cache == nil { + cache = map[string][]int{} + } + if cache[name] == nil { + cache[name] = ids + } else { + cache[name] = append(cache[name], ids...) + } + // marshal config data to yaml + configDataYaml, err := yaml.Marshal(configData) + if err != nil { + return err + } + // write config data to file + err = ioutil.WriteFile(configFilePath, configDataYaml, os.ModePerm) + if err != nil { + return err + } + return nil +} + +func loadFromCache(name string) ([]int, error) { + configData, err := getConfig(configFilePath) + if err != nil { + return nil, err + } + if configData.Cache != nil { + if configData.Cache[name] != nil { + return configData.Cache[name], nil + } + } + return []int{}, nil +} + +func getConfig(path string) (Config, error) { + configFile, err := ioutil.ReadFile(configFilePath) + var configData Config + if err != nil { + return configData, err + } + err = yaml.Unmarshal(configFile, &configData) + if err != nil { + log.Fatal(err) + } + return configData, nil +} + +type Config struct { + Controller string `yaml:"controller"` + ListFile string `yaml:"listFile"` + Cache map[string][]int `yaml:"cache"` +} + +func main() { + + // read config file + homePath := os.Getenv("XDG_CONFIG_HOME") + if homePath == "" { + homePath = os.Getenv("HOME") + } + configFilePath = filepath.Join(homePath, ".config", "mrva", "config.yml") + if _, err := os.Stat(configFilePath); os.IsNotExist(err) { + // create config file if it doesn't exist + // since we will use it for the name/ids cache + err := os.MkdirAll(filepath.Dir(configFilePath), os.ModePerm) + if err != nil { + log.Println("Failed to create config file directory") + } + // create empty file at configFilePath + configFile, err := os.Create(configFilePath) + if err != nil { + log.Fatal(err, "Failed to create config file") + } + configFile.Close() + } + configData, err := getConfig(configFilePath) + if err != nil { + log.Fatal(err) + } + if configData.Controller != "" { + controller = configData.Controller + } + if configData.ListFile != "" { + listFile = configData.ListFile + } + + flag.Parse() + args := flag.Args() + if len(args) == 0 { + log.Fatal("Please specify a subcommand.") + } + cmd, args := args[0], args[1:] + + switch cmd { + case "submit": + submit(args) + case "download": + download(args) + default: + log.Fatalf("Unrecognized command %q. "+ + "Command must be one of: submit, download", cmd) + } +} + +func submit(args []string) { + flag := flag.NewFlagSet("mrva submit", flag.ExitOnError) + queryFileFlag := flag.String("query", "", "Path to query file") + controllerFlag := flag.String("controller", "", "MRVA controller repository (overrides config file)") + listFileFlag := flag.String("list-file", "", "Path to repo list file (overrides config file)") + listFlag := flag.String("list", "", "Name of repo list") + langFlag := flag.String("lang", "", "DB language") + nameFlag := flag.String("name", "", "Name of run (optional)") + + flag.Usage = func() { + fmt.Fprintf(os.Stderr, `gh mrva - submit and download CodeQL queries from MRVA +Usage: + gh mrva submit --controller --lang [--name ] --list-file --list --query + +Flags: +`) + flag.PrintDefaults() + } + flag.Parse(args) + + // set global variables + if *langFlag != "" { + language = *langFlag + } + if *nameFlag != "" { + runName = *nameFlag + } + if *controllerFlag != "" { + controller = *controllerFlag + } + if *listFileFlag != "" { + listFile = *listFileFlag + } + + if controller == "" || language == "" || listFile == "" || *listFlag == "" || *queryFileFlag == "" { + flag.Usage() + os.Exit(1) + } + + repositories, err := resolveRepositories(listFile, *listFlag) + if err != nil { + log.Fatal(err) + } + fmt.Printf("Requesting run for %d repositories from %s:%s\n", len(repositories), listFile, *listFlag) + + encodedBundle, err := generateQueryPack(*queryFileFlag) + if err != nil { + log.Fatal(err) + } + fmt.Println("Generated encoded bundle") + + var chunks [][]string + for i := 0; i < len(repositories); i += MAX_MRVA_REPOSITORIES { + end := i + MAX_MRVA_REPOSITORIES + if end > len(repositories) { + end = len(repositories) + } + chunks = append(chunks, repositories[i:end]) + } + var ids []int + for _, chunk := range chunks { + id, err := submitRun(chunk, encodedBundle) + if err != nil { + log.Fatal(err) + } + ids = append(ids, id) + } + fmt.Printf("Submitted run %v\n", ids) + if runName != "" { + err = saveInCache(runName, ids) + if err != nil { + log.Fatal(err) + } + } +} + +func download(args []string) { + flag := flag.NewFlagSet("mrva submit", flag.ExitOnError) + runFlag := flag.Int("run", 0, "MRVA run ID") + outputDirFlag := flag.String("output-dir", "", "Output directory") + downloadDBsFlag := flag.Bool("download-dbs", false, "Download databases (optional)") + controllerFlag := flag.String("controller", "", "MRVA controller repository (overrides config file)") + langFlag := flag.String("lang", "", "DB language") + nameFlag := flag.String("name", "", "Name of run (optional)") + + flag.Usage = func() { + fmt.Fprintf(os.Stderr, `gh mrva - submit and download CodeQL queries from MRVA +Usage: + gh mrva download --run --lang --controller --output-dir [--name ] [--download-dbs] + +Flags: +`) + flag.PrintDefaults() + } + + flag.Parse(args) + + // set global variables + if *langFlag != "" { + language = *langFlag + } + if *nameFlag != "" { + runName = *nameFlag + } + if *controllerFlag != "" { + controller = *controllerFlag + } + + if controller == "" || language == "" || (*runFlag == 0 && runName == "") || *outputDirFlag == "" { + flag.Usage() + os.Exit(1) + } + + // if outputDirFlag does not exist, create it + if _, err := os.Stat(*outputDirFlag); os.IsNotExist(err) { + err := os.MkdirAll(*outputDirFlag, os.ModePerm) + if err != nil { + log.Fatal(err) + } + } + + runIds := []int{} + if *runFlag > 0 { + runIds = []int{*runFlag} + } else if runName != "" { + ids, err := loadFromCache(runName) + if err != nil { + log.Fatal(err) + } + if len(ids) > 0 { + runIds = ids + } + } + + for _, runId := range runIds { + fmt.Printf("Downloading MRVA results for %s (%d)\n", controller, runId) + // check if the run is complete + runDetails, err := getRunDetails(runId) + fmt.Printf("Status: %v\n", runDetails["status"]) + if err != nil { + log.Fatal(err) + } + if runDetails["status"] == "in_progress" { + log.Printf("Run %d is not complete yet. Please try again later.", runId) + return + } + for _, r := range runDetails["scanned_repositories"].([]interface{}) { + repo := r.(map[string]interface{}) + result_count := repo["result_count"] + repoInfo := repo["repository"].(map[string]interface{}) + nwo := repoInfo["full_name"].(string) + if result_count != nil && result_count.(float64) > 0 { + fmt.Printf("Repo %s has %d results\n", nwo, int(result_count.(float64))) + resultPath := filepath.Join(*outputDirFlag, fmt.Sprintf("%s.sarif", strings.Replace(nwo, "/", "_", -1))) + + if _, err := os.Stat(resultPath); errors.Is(err, os.ErrNotExist) { + + // download artifact (BQRS or SARIF) + fmt.Printf("Downloading results for %s\n", repoInfo["full_name"]) + runRepositoryDetails, err := getRunRepositoryDetails(runId, nwo) + if err != nil { + log.Fatal(err) + } + // download the results + artifactPath, err := downloadArtifact(runRepositoryDetails["artifact_url"].(string), *outputDirFlag) + if err != nil { + log.Fatal(err) + } + fmt.Printf("Artifact path: %s\n", artifactPath) + } + if *downloadDBsFlag { + // download database + targetPath := filepath.Join(*outputDirFlag, fmt.Sprintf("%s_%s_db.zip", strings.Replace(nwo, "/", "_", -1), language)) + if _, err := os.Stat(targetPath); errors.Is(err, os.ErrNotExist) { + fmt.Printf("Downloading database for %s\n", nwo) + err = downloadDatabase(nwo, language, targetPath) + if err != nil { + log.Fatal(err) + } + fmt.Printf("Database path: %s\n", targetPath) + } + } + } + } + } +} diff --git a/tests/.cache/data/15/a4/15a4cf726148fed00e9291fa46dd48edf9507125 b/tests/.cache/data/15/a4/15a4cf726148fed00e9291fa46dd48edf9507125 new file mode 100644 index 0000000..eb2f67b --- /dev/null +++ b/tests/.cache/data/15/a4/15a4cf726148fed00e9291fa46dd48edf9507125 @@ -0,0 +1,20 @@ + # Precompiled CodeQL query +--- +format: +- 202210190 +creator: "2.12.4" +name: "query.ql" +dbscheme: "934bf10b4bd34cf648893efcd1d0d7be9471d39f" +stages: +- cached: "d366cddf0b9c37dfe2b2b40f6b96537096e08d00" + debugInfo: "fecaa121855b37cb5060daceb1adc2e02056f801" +- cached: "248f1f782c7cbe975da92bc515b37371b1c1136b" + debugInfo: "fecaa121855b37cb5060daceb1adc2e02056f801" +- cached: "cffbc7d7aa728a15b1e164d22564f43a1d232b17" + debugInfo: "fecaa121855b37cb5060daceb1adc2e02056f801" +- cached: "4d40c42c794541aa678d01096f984031c66b3e68" + debugInfo: "fecaa121855b37cb5060daceb1adc2e02056f801" +results: + '#select': + resultArranger: "1\"ma\"ei12" +inputsDigest: "15a4cf726148fed00e9291fa46dd48edf9507125" diff --git a/tests/.cache/data/24/8f/248f1f782c7cbe975da92bc515b37371b1c1136b b/tests/.cache/data/24/8f/248f1f782c7cbe975da92bc515b37371b1c1136b new file mode 100644 index 0000000..cdd9eb5 Binary files /dev/null and b/tests/.cache/data/24/8f/248f1f782c7cbe975da92bc515b37371b1c1136b differ diff --git a/tests/.cache/data/4d/40/4d40c42c794541aa678d01096f984031c66b3e68 b/tests/.cache/data/4d/40/4d40c42c794541aa678d01096f984031c66b3e68 new file mode 100644 index 0000000..3a79b24 Binary files /dev/null and b/tests/.cache/data/4d/40/4d40c42c794541aa678d01096f984031c66b3e68 differ diff --git a/tests/.cache/data/b2/e5/b2e5a6727f6474d5add4e280b09df9b3eb2a7a02 b/tests/.cache/data/b2/e5/b2e5a6727f6474d5add4e280b09df9b3eb2a7a02 new file mode 100644 index 0000000..4036d73 --- /dev/null +++ b/tests/.cache/data/b2/e5/b2e5a6727f6474d5add4e280b09df9b3eb2a7a02 @@ -0,0 +1,20 @@ + # Precompiled CodeQL query +--- +format: +- 202210190 +creator: "2.12.4" +name: "query.ql" +dbscheme: "934bf10b4bd34cf648893efcd1d0d7be9471d39f" +stages: +- cached: "d366cddf0b9c37dfe2b2b40f6b96537096e08d00" + debugInfo: "fecaa121855b37cb5060daceb1adc2e02056f801" +- cached: "248f1f782c7cbe975da92bc515b37371b1c1136b" + debugInfo: "fecaa121855b37cb5060daceb1adc2e02056f801" +- cached: "cffbc7d7aa728a15b1e164d22564f43a1d232b17" + debugInfo: "fecaa121855b37cb5060daceb1adc2e02056f801" +- cached: "4d40c42c794541aa678d01096f984031c66b3e68" + debugInfo: "fecaa121855b37cb5060daceb1adc2e02056f801" +results: + '#select': + resultArranger: "1\"ma\"ei12" +inputsDigest: "b2e5a6727f6474d5add4e280b09df9b3eb2a7a02" diff --git a/tests/.cache/data/cf/fb/cffbc7d7aa728a15b1e164d22564f43a1d232b17 b/tests/.cache/data/cf/fb/cffbc7d7aa728a15b1e164d22564f43a1d232b17 new file mode 100644 index 0000000..78b0a68 Binary files /dev/null and b/tests/.cache/data/cf/fb/cffbc7d7aa728a15b1e164d22564f43a1d232b17 differ diff --git a/tests/.cache/data/d3/66/d366cddf0b9c37dfe2b2b40f6b96537096e08d00 b/tests/.cache/data/d3/66/d366cddf0b9c37dfe2b2b40f6b96537096e08d00 new file mode 100644 index 0000000..46c998f Binary files /dev/null and b/tests/.cache/data/d3/66/d366cddf0b9c37dfe2b2b40f6b96537096e08d00 differ diff --git a/tests/.cache/data/fe/ca/fecaa121855b37cb5060daceb1adc2e02056f801 b/tests/.cache/data/fe/ca/fecaa121855b37cb5060daceb1adc2e02056f801 new file mode 100644 index 0000000..39645da Binary files /dev/null and b/tests/.cache/data/fe/ca/fecaa121855b37cb5060daceb1adc2e02056f801 differ diff --git a/tests/.cache/lock b/tests/.cache/lock new file mode 100644 index 0000000..e69de29 diff --git a/tests/.cache/size b/tests/.cache/size new file mode 100644 index 0000000..1fa9df4 Binary files /dev/null and b/tests/.cache/size differ diff --git a/tests/query.ql b/tests/query.ql new file mode 100644 index 0000000..94d2e7c --- /dev/null +++ b/tests/query.ql @@ -0,0 +1,4 @@ +import java +from MethodAccess ma +where ma.getMethod().getName().matches("a%") +select ma diff --git a/tests/repos.json b/tests/repos.json new file mode 100644 index 0000000..13cee76 --- /dev/null +++ b/tests/repos.json @@ -0,0 +1,3 @@ +{ + "sample": ["apache/logging-log4j2"] +}