From b9081b1945157d87a5b4413a2f57092552935070 Mon Sep 17 00:00:00 2001 From: Michael Hohn Date: Fri, 31 May 2024 08:24:09 -0700 Subject: [PATCH] wip: convert run-analysis.sh to golang version --- README.md | 7 + cmd/agent/main.go | 1 + cmd/runner/main.go | 0 cmd/server/bin/run-analysis.sh | 58 -------- pkg/agent/agent.go | 234 +++++++++++++++++++++++++++------ 5 files changed, 199 insertions(+), 101 deletions(-) create mode 100644 cmd/agent/main.go delete mode 100644 cmd/runner/main.go delete mode 100755 cmd/server/bin/run-analysis.sh diff --git a/README.md b/README.md index e69de29..42576ff 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,7 @@ +# Overview + +TODO diagram + +TODO note: NO package init() functions +Dynamic behaviour must be explicit + \ No newline at end of file diff --git a/cmd/agent/main.go b/cmd/agent/main.go new file mode 100644 index 0000000..4883155 --- /dev/null +++ b/cmd/agent/main.go @@ -0,0 +1 @@ +package agent diff --git a/cmd/runner/main.go b/cmd/runner/main.go deleted file mode 100644 index e69de29..0000000 diff --git a/cmd/server/bin/run-analysis.sh b/cmd/server/bin/run-analysis.sh deleted file mode 100755 index 7e98e60..0000000 --- a/cmd/server/bin/run-analysis.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -x -e -#* Minimal setup to run analysis using information provided by a request - -#* Take output saved by the server -QUERYPACKID=$1 -shift -QUERYLANGUAGE=$1 -shift - -# and -DBOWNER=$1 -shift -DBREPO=$1 - -# FIXME Provide this via environment or explicit argument -GMSROOT=/Users/hohn/local/ghes-mirva-server -GMSROOT=/Users/hohn/work-gh/mrva/mrvacommander/cmd/server - -#* Set up derived paths -DBPATH=$GMSROOT/var/codeql/dbs/$DBOWNER/$DBREPO -DBZIP=$GMSROOT/codeql/dbs/$DBOWNER/$DBREPO/${DBOWNER}_${DBREPO}_db.zip -DBEXTRACT=$GMSROOT/var/codeql/dbs/$DBOWNER/$DBREPO - -QUERYPACK=$GMSROOT/var/codeql/querypacks/qp-$QUERYPACKID.tgz -QUERYEXTRACT=$GMSROOT/var/codeql/querypacks/qp-$QUERYPACKID - -QUERYOUTD=$GMSROOT/var/codeql/sarif/localrun/$DBOWNER/$DBREPO -QUERYOUTF=$QUERYOUTD/${DBOWNER}_${DBREPO}.sarif - -#* Prep work before running the command - -#** Extract database -mkdir -p $DBEXTRACT && cd $DBEXTRACT -unzip -o -q $DBZIP - -# Extract query pack -mkdir -p $QUERYEXTRACT && cd $QUERYEXTRACT -tar zxf $QUERYPACK - -#** Prepare target directory -mkdir -p $QUERYOUTD - -#* run database analyze -cd $GMSROOT -codeql database analyze --format=sarif-latest --rerun \ - --output $QUERYOUTF \ - -j8 \ - -- $DBPATH $QUERYEXTRACT - -#* report result -printf "run-analysis-output in %s\n" $QUERYOUTF - -# TODO Is the bqrs really necessary? It can also be found by the go code at this point. -# TODO The name of the query is only found in the query pack itself and would have -# to be extracted from there. - -# BQRSPATH=$GMSROOT/var/codeql/dbs/$DBOWNER/$DBREPO/$LANGUAGE/results/codeql-remote/query/ -# printf "run-analysis-bqrs in %s\n" $QUERYOUTF diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 179b362..07f1bd4 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -1,19 +1,21 @@ package agent import ( - "bufio" - "bytes" - "log/slog" - "mrvacommander/pkg/common" "mrvacommander/pkg/queue" "mrvacommander/pkg/storage" + "log/slog" + + "archive/tar" + "archive/zip" + "compress/gzip" + "fmt" + "io" + "path/filepath" + "os" "os/exec" - "path" - "strconv" - "strings" ) type RunnerSingle struct { @@ -37,50 +39,196 @@ func (r *RunnerSingle) worker(wid int) { slog.Debug("Picking up job", "job", job, "worker", wid) - cwd, err := os.Getwd() - if err != nil { - slog.Error("RunJob: cwd problem: ", "error", err) - continue - } - slog.Debug("Analysis: running", "job", job) storage.SetStatus(job.QueryPackId, job.ORL, common.StatusQueued) - cmd := exec.Command(path.Join(cwd, "bin", "run-analysis.sh"), - strconv.FormatInt(int64(job.QueryPackId), 10), - job.QueryLanguage, job.ORL.Owner, job.ORL.Repo) - out, err := cmd.CombinedOutput() + resultFile, err := r.RunAnalysis(job) if err != nil { - slog.Error("Analysis command failed: exit code: ", "error", err, "job", job) - slog.Error("Analysis command failed: ", "job", job, "output", out) - storage.SetStatus(job.QueryPackId, job.ORL, common.StatusError) continue } + slog.Debug("Analysis run finished", "job", job) - // Get the SARIF ouput location - sr := bufio.NewScanner(bytes.NewReader(out)) - sr.Split(bufio.ScanLines) - for { - more := sr.Scan() - if !more { - slog.Error("Analysis run failed to report result: ", "output", out) - break - } - fields := strings.Fields(sr.Text()) - if len(fields) >= 3 { - if fields[0] == "run-analysis-output" { - slog.Debug("Analysis run successful: ", "job", job, "location", fields[2]) - res := common.AnalyzeResult{ - RunAnalysisSARIF: fields[2], // Abs. path from run-analysis.sh - RunAnalysisBQRS: "", // FIXME? see note in run-analysis.sh - } - r.queue.Results() <- res - storage.SetStatus(job.QueryPackId, job.ORL, common.StatusSuccess) - storage.SetResult(job.QueryPackId, job.ORL, res) - break - } - } + res := common.AnalyzeResult{ + RunAnalysisSARIF: resultFile, + RunAnalysisBQRS: "", // FIXME ? } + r.queue.Results() <- res + storage.SetStatus(job.QueryPackId, job.ORL, common.StatusSuccess) + storage.SetResult(job.QueryPackId, job.ORL, res) + } } + +func (r *RunnerSingle) RunAnalysis(job common.AnalyzeJob) (string, error) { + // TODO Add multi-language tests including queryLanguage + // queryPackID, queryLanguage, dbOwner, dbRepo := + // job.QueryPackId, job.QueryLanguage, job.ORL.Owner, job.ORL.Repo + queryPackID, dbOwner, dbRepo := + job.QueryPackId, job.ORL.Owner, job.ORL.Repo + + // FIXME Provide this via environment or explicit argument + gmsRoot := "/Users/hohn/work-gh/mrva/mrvacommander/cmd/server" + + // Set up derived paths + dbPath := filepath.Join(gmsRoot, "var/codeql/dbs", dbOwner, dbRepo) + dbZip := filepath.Join(gmsRoot, "codeql/dbs", dbOwner, dbRepo, + fmt.Sprintf("%s_%s_db.zip", dbOwner, dbRepo)) + dbExtract := filepath.Join(gmsRoot, "var/codeql/dbs", dbOwner, dbRepo) + + queryPack := filepath.Join(gmsRoot, + "var/codeql/querypacks", fmt.Sprintf("qp-%d.tgz", queryPackID)) + queryExtract := filepath.Join(gmsRoot, + "var/codeql/querypacks", fmt.Sprintf("qp-%d", queryPackID)) + + queryOutDir := filepath.Join(gmsRoot, + "var/codeql/sarif/localrun", dbOwner, dbRepo) + queryOutFile := filepath.Join(queryOutDir, + fmt.Sprintf("%s_%s.sarif", dbOwner, dbRepo)) + + // Prepare directory, extract database + if err := os.MkdirAll(dbExtract, 0755); err != nil { + slog.Error("Failed to create DB directory %s: %v", dbExtract, err) + return "", err + } + + if err := unzipFile(dbZip, dbExtract); err != nil { + slog.Error("Failed to unzip DB %s: %v", dbZip, err) + return "", err + } + + // Prepare directory, extract query pack + if err := os.MkdirAll(queryExtract, 0755); err != nil { + slog.Error("Failed to create query pack directory %s: %v", queryExtract, err) + return "", err + } + + if err := untarGz(queryPack, queryExtract); err != nil { + slog.Error("Failed to extract querypack %s: %v", queryPack, err) + return "", err + } + + // Prepare query result directory + if err := os.MkdirAll(queryOutDir, 0755); err != nil { + slog.Error("Failed to create query result directory %s: %v", queryOutDir, err) + return "", err + } + + // Run database analyze + cmd := exec.Command("codeql", "database", "analyze", + "--format=sarif-latest", "--rerun", "--output", queryOutFile, + "-j8", dbPath, queryExtract) + cmd.Dir = gmsRoot + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + slog.Error("codeql database analyze failed:", "error", err, "job", job) + storage.SetStatus(job.QueryPackId, job.ORL, common.StatusError) + return "", err + } + + // Return result path + return queryOutFile, nil +} + +// unzipFile extracts a zip file to the specified destination +func unzipFile(zipFile, dest string) error { + r, err := zip.OpenReader(zipFile) + if err != nil { + return err + } + defer r.Close() + + for _, f := range r.File { + fPath := filepath.Join(dest, f.Name) + if f.FileInfo().IsDir() { + if err := os.MkdirAll(fPath, os.ModePerm); err != nil { + return err + } + continue + } + + if err := os.MkdirAll(filepath.Dir(fPath), os.ModePerm); err != nil { + return err + } + + outFile, err := os.OpenFile(fPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + return err + } + + rc, err := f.Open() + if err != nil { + outFile.Close() + return err + } + + _, err = io.Copy(outFile, rc) + + outFile.Close() + rc.Close() + + if err != nil { + return err + } + } + return nil +} + +// untarGz extracts a tar.gz file to the specified destination. +func untarGz(tarGzFile, dest string) error { + file, err := os.Open(tarGzFile) + if err != nil { + return err + } + defer file.Close() + + gzr, err := gzip.NewReader(file) + if err != nil { + return err + } + defer gzr.Close() + + return untar(gzr, dest) +} + +// untar extracts a tar archive to the specified destination. +func untar(r io.Reader, dest string) error { + tr := tar.NewReader(r) + + for { + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + + fPath := filepath.Join(dest, header.Name) + if header.Typeflag == tar.TypeDir { + if err := os.MkdirAll(fPath, os.ModePerm); err != nil { + return err + } + } else { + if err := os.MkdirAll(filepath.Dir(fPath), os.ModePerm); err != nil { + return err + } + + outFile, err := os.OpenFile(fPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(header.Mode)) + if err != nil { + return err + } + + if _, err := io.Copy(outFile, tr); err != nil { + outFile.Close() + return err + } + + outFile.Close() + } + } + + return nil +}