wip: convert run-analysis.sh to golang version
This commit is contained in:
Michael Hohn
committed by
=Michael Hohn
=Michael Hohn
parent
75d0d312db
commit
b9081b1945
@@ -0,0 +1,7 @@
|
||||
# Overview
|
||||
|
||||
TODO diagram
|
||||
|
||||
TODO note: NO package init() functions
|
||||
Dynamic behaviour must be explicit
|
||||
|
||||
1
cmd/agent/main.go
Normal file
1
cmd/agent/main.go
Normal file
@@ -0,0 +1 @@
|
||||
package agent
|
||||
@@ -1,58 +0,0 @@
|
||||
#!/bin/bash -x -e
|
||||
#* Minimal setup to run analysis using information provided by a request
|
||||
|
||||
#* Take output saved by the server
|
||||
QUERYPACKID=$1
|
||||
shift
|
||||
QUERYLANGUAGE=$1
|
||||
shift
|
||||
|
||||
# and
|
||||
DBOWNER=$1
|
||||
shift
|
||||
DBREPO=$1
|
||||
|
||||
# FIXME Provide this via environment or explicit argument
|
||||
GMSROOT=/Users/hohn/local/ghes-mirva-server
|
||||
GMSROOT=/Users/hohn/work-gh/mrva/mrvacommander/cmd/server
|
||||
|
||||
#* Set up derived paths
|
||||
DBPATH=$GMSROOT/var/codeql/dbs/$DBOWNER/$DBREPO
|
||||
DBZIP=$GMSROOT/codeql/dbs/$DBOWNER/$DBREPO/${DBOWNER}_${DBREPO}_db.zip
|
||||
DBEXTRACT=$GMSROOT/var/codeql/dbs/$DBOWNER/$DBREPO
|
||||
|
||||
QUERYPACK=$GMSROOT/var/codeql/querypacks/qp-$QUERYPACKID.tgz
|
||||
QUERYEXTRACT=$GMSROOT/var/codeql/querypacks/qp-$QUERYPACKID
|
||||
|
||||
QUERYOUTD=$GMSROOT/var/codeql/sarif/localrun/$DBOWNER/$DBREPO
|
||||
QUERYOUTF=$QUERYOUTD/${DBOWNER}_${DBREPO}.sarif
|
||||
|
||||
#* Prep work before running the command
|
||||
|
||||
#** Extract database
|
||||
mkdir -p $DBEXTRACT && cd $DBEXTRACT
|
||||
unzip -o -q $DBZIP
|
||||
|
||||
# Extract query pack
|
||||
mkdir -p $QUERYEXTRACT && cd $QUERYEXTRACT
|
||||
tar zxf $QUERYPACK
|
||||
|
||||
#** Prepare target directory
|
||||
mkdir -p $QUERYOUTD
|
||||
|
||||
#* run database analyze
|
||||
cd $GMSROOT
|
||||
codeql database analyze --format=sarif-latest --rerun \
|
||||
--output $QUERYOUTF \
|
||||
-j8 \
|
||||
-- $DBPATH $QUERYEXTRACT
|
||||
|
||||
#* report result
|
||||
printf "run-analysis-output in %s\n" $QUERYOUTF
|
||||
|
||||
# TODO Is the bqrs really necessary? It can also be found by the go code at this point.
|
||||
# TODO The name of the query is only found in the query pack itself and would have
|
||||
# to be extracted from there.
|
||||
|
||||
# BQRSPATH=$GMSROOT/var/codeql/dbs/$DBOWNER/$DBREPO/$LANGUAGE/results/codeql-remote/query/
|
||||
# printf "run-analysis-bqrs in %s\n" $QUERYOUTF
|
||||
@@ -1,19 +1,21 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"log/slog"
|
||||
|
||||
"mrvacommander/pkg/common"
|
||||
"mrvacommander/pkg/queue"
|
||||
"mrvacommander/pkg/storage"
|
||||
|
||||
"log/slog"
|
||||
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type RunnerSingle struct {
|
||||
@@ -37,50 +39,196 @@ func (r *RunnerSingle) worker(wid int) {
|
||||
|
||||
slog.Debug("Picking up job", "job", job, "worker", wid)
|
||||
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
slog.Error("RunJob: cwd problem: ", "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
slog.Debug("Analysis: running", "job", job)
|
||||
storage.SetStatus(job.QueryPackId, job.ORL, common.StatusQueued)
|
||||
cmd := exec.Command(path.Join(cwd, "bin", "run-analysis.sh"),
|
||||
strconv.FormatInt(int64(job.QueryPackId), 10),
|
||||
job.QueryLanguage, job.ORL.Owner, job.ORL.Repo)
|
||||
|
||||
out, err := cmd.CombinedOutput()
|
||||
resultFile, err := r.RunAnalysis(job)
|
||||
if err != nil {
|
||||
slog.Error("Analysis command failed: exit code: ", "error", err, "job", job)
|
||||
slog.Error("Analysis command failed: ", "job", job, "output", out)
|
||||
storage.SetStatus(job.QueryPackId, job.ORL, common.StatusError)
|
||||
continue
|
||||
}
|
||||
|
||||
slog.Debug("Analysis run finished", "job", job)
|
||||
|
||||
// Get the SARIF ouput location
|
||||
sr := bufio.NewScanner(bytes.NewReader(out))
|
||||
sr.Split(bufio.ScanLines)
|
||||
for {
|
||||
more := sr.Scan()
|
||||
if !more {
|
||||
slog.Error("Analysis run failed to report result: ", "output", out)
|
||||
break
|
||||
}
|
||||
fields := strings.Fields(sr.Text())
|
||||
if len(fields) >= 3 {
|
||||
if fields[0] == "run-analysis-output" {
|
||||
slog.Debug("Analysis run successful: ", "job", job, "location", fields[2])
|
||||
res := common.AnalyzeResult{
|
||||
RunAnalysisSARIF: fields[2], // Abs. path from run-analysis.sh
|
||||
RunAnalysisBQRS: "", // FIXME? see note in run-analysis.sh
|
||||
RunAnalysisSARIF: resultFile,
|
||||
RunAnalysisBQRS: "", // FIXME ?
|
||||
}
|
||||
r.queue.Results() <- res
|
||||
storage.SetStatus(job.QueryPackId, job.ORL, common.StatusSuccess)
|
||||
storage.SetResult(job.QueryPackId, job.ORL, res)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func (r *RunnerSingle) RunAnalysis(job common.AnalyzeJob) (string, error) {
|
||||
// TODO Add multi-language tests including queryLanguage
|
||||
// queryPackID, queryLanguage, dbOwner, dbRepo :=
|
||||
// job.QueryPackId, job.QueryLanguage, job.ORL.Owner, job.ORL.Repo
|
||||
queryPackID, dbOwner, dbRepo :=
|
||||
job.QueryPackId, job.ORL.Owner, job.ORL.Repo
|
||||
|
||||
// FIXME Provide this via environment or explicit argument
|
||||
gmsRoot := "/Users/hohn/work-gh/mrva/mrvacommander/cmd/server"
|
||||
|
||||
// Set up derived paths
|
||||
dbPath := filepath.Join(gmsRoot, "var/codeql/dbs", dbOwner, dbRepo)
|
||||
dbZip := filepath.Join(gmsRoot, "codeql/dbs", dbOwner, dbRepo,
|
||||
fmt.Sprintf("%s_%s_db.zip", dbOwner, dbRepo))
|
||||
dbExtract := filepath.Join(gmsRoot, "var/codeql/dbs", dbOwner, dbRepo)
|
||||
|
||||
queryPack := filepath.Join(gmsRoot,
|
||||
"var/codeql/querypacks", fmt.Sprintf("qp-%d.tgz", queryPackID))
|
||||
queryExtract := filepath.Join(gmsRoot,
|
||||
"var/codeql/querypacks", fmt.Sprintf("qp-%d", queryPackID))
|
||||
|
||||
queryOutDir := filepath.Join(gmsRoot,
|
||||
"var/codeql/sarif/localrun", dbOwner, dbRepo)
|
||||
queryOutFile := filepath.Join(queryOutDir,
|
||||
fmt.Sprintf("%s_%s.sarif", dbOwner, dbRepo))
|
||||
|
||||
// Prepare directory, extract database
|
||||
if err := os.MkdirAll(dbExtract, 0755); err != nil {
|
||||
slog.Error("Failed to create DB directory %s: %v", dbExtract, err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
if err := unzipFile(dbZip, dbExtract); err != nil {
|
||||
slog.Error("Failed to unzip DB %s: %v", dbZip, err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Prepare directory, extract query pack
|
||||
if err := os.MkdirAll(queryExtract, 0755); err != nil {
|
||||
slog.Error("Failed to create query pack directory %s: %v", queryExtract, err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
if err := untarGz(queryPack, queryExtract); err != nil {
|
||||
slog.Error("Failed to extract querypack %s: %v", queryPack, err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Prepare query result directory
|
||||
if err := os.MkdirAll(queryOutDir, 0755); err != nil {
|
||||
slog.Error("Failed to create query result directory %s: %v", queryOutDir, err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Run database analyze
|
||||
cmd := exec.Command("codeql", "database", "analyze",
|
||||
"--format=sarif-latest", "--rerun", "--output", queryOutFile,
|
||||
"-j8", dbPath, queryExtract)
|
||||
cmd.Dir = gmsRoot
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
slog.Error("codeql database analyze failed:", "error", err, "job", job)
|
||||
storage.SetStatus(job.QueryPackId, job.ORL, common.StatusError)
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Return result path
|
||||
return queryOutFile, nil
|
||||
}
|
||||
|
||||
// unzipFile extracts a zip file to the specified destination
|
||||
func unzipFile(zipFile, dest string) error {
|
||||
r, err := zip.OpenReader(zipFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
for _, f := range r.File {
|
||||
fPath := filepath.Join(dest, f.Name)
|
||||
if f.FileInfo().IsDir() {
|
||||
if err := os.MkdirAll(fPath, os.ModePerm); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(fPath), os.ModePerm); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
outFile, err := os.OpenFile(fPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
outFile.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = io.Copy(outFile, rc)
|
||||
|
||||
outFile.Close()
|
||||
rc.Close()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// untarGz extracts a tar.gz file to the specified destination.
|
||||
func untarGz(tarGzFile, dest string) error {
|
||||
file, err := os.Open(tarGzFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
gzr, err := gzip.NewReader(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer gzr.Close()
|
||||
|
||||
return untar(gzr, dest)
|
||||
}
|
||||
|
||||
// untar extracts a tar archive to the specified destination.
|
||||
func untar(r io.Reader, dest string) error {
|
||||
tr := tar.NewReader(r)
|
||||
|
||||
for {
|
||||
header, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fPath := filepath.Join(dest, header.Name)
|
||||
if header.Typeflag == tar.TypeDir {
|
||||
if err := os.MkdirAll(fPath, os.ModePerm); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if err := os.MkdirAll(filepath.Dir(fPath), os.ModePerm); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
outFile, err := os.OpenFile(fPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(header.Mode))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := io.Copy(outFile, tr); err != nil {
|
||||
outFile.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
outFile.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user