diff --git a/cmd/server/main.go b/cmd/server/main.go index 0901ea0..80f6eef 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -29,7 +29,7 @@ func main() { log.Printf("Usage of %s:\n", os.Args[0]) flag.PrintDefaults() log.Println("\nExamples:") - log.Println(" go run main.go --loglevel=Debug --mode=container") + log.Println(" go run main.go --loglevel=debug --mode=container") } // Parse the flags @@ -68,16 +68,21 @@ func main() { switch *mode { case "standalone": // Assemble single-process version + sq := queue.NewQueueSingle(2) // FIXME take value from configuration + sc := server.NewCommanderSingle(nil, sq) + sl := logger.NewLoggerSingle() + ss := storage.NewStorageSingle(config.Storage.StartingID) + sr := agent.NewRunnerSingle(2, sq) // FIXME take value from configuration + state := server.State{ - Commander: &server.CommanderSingle{}, - Logger: &logger.LoggerSingle{}, - Queue: &queue.QueueSingle{}, - Storage: &storage.StorageSingle{CurrentID: config.Storage.StartingID}, - Runner: &agent.RunnerSingle{}, + Commander: sc, + Logger: sl, + Queue: sq, + Storage: ss, + Runner: sr, } - main := &server.CommanderSingle{} - main.Setup(&state) - main.Run() + + sc.Setup(&state) // sc is part of state and dereferences it case "container": // Assemble cccontainer diff --git a/cmd/server/run-analysis.sh b/cmd/server/run-analysis.sh new file mode 100755 index 0000000..076fd00 --- /dev/null +++ b/cmd/server/run-analysis.sh @@ -0,0 +1,57 @@ +#!/bin/bash -x -e +#* Minimal setup to run analysis using information provided by a request + +#* Take output saved by the server +QUERYPACKID=$1 +shift +QUERYLANGUAGE=$1 +shift + +# and +DBOWNER=$1 +shift +DBREPO=$1 + +GMSROOT=/Users/hohn/local/ghes-mirva-server + +#* Set up derived paths +DBPATH=$GMSROOT/var/codeql/dbs/$DBOWNER/$DBREPO +DBZIP=$GMSROOT/codeql/dbs/$DBOWNER/$DBREPO/${DBOWNER}_${DBREPO}_db.zip +DBEXTRACT=$GMSROOT/var/codeql/dbs/$DBOWNER/$DBREPO + +QUERYPACK=$GMSROOT/var/codeql/querypacks/qp-$QUERYPACKID.tgz +QUERYEXTRACT=$GMSROOT/var/codeql/querypacks/qp-$QUERYPACKID + +QUERYOUTD=$GMSROOT/var/codeql/sarif/localrun/$DBOWNER/$DBREPO +QUERYOUTF=$QUERYOUTD/${DBOWNER}_${DBREPO}.sarif + +#* Prep work before running the command + +#** Extract database +mkdir -p $DBEXTRACT && cd $DBEXTRACT +unzip -o -q $DBZIP +DBINFIX=`\ls | head -1` # Could be cpp, codeql_db, or whatever + +# Extract query pack +mkdir -p $QUERYEXTRACT && cd $QUERYEXTRACT +tar zxf $QUERYPACK + +#** Prepare target directory +mkdir -p $QUERYOUTD + +#* run database analyze +cd $GMSROOT +codeql database analyze --format=sarif-latest --rerun \ + --output $QUERYOUTF \ + -j8 \ + -- $DBPATH/$DBINFIX $QUERYEXTRACT + +#* report result +# var/codeql/sarif/localrun/google/flatbuffers/google_flatbuffers.sarif +printf "run-analysis-output in %s\n" $QUERYOUTF +# TODO Is the bqrs really necessary? It can also be found by the go code at this point. +# The name of the query is only found in the query pack itself and would have to be +# extracted from there. +# var/codeql/dbs/google/flatbuffers/cpp/results/codeql-remote/query/FlatBuffersFunc.bqrs +# BQRSPATH=$GMSROOT/var/codeql/dbs/$DBOWNER/$DBREPO/$LANGUAGE/results/codeql-remote/query/ +# printf "run-analysis-bqrs in %s\n" $QUERYOUTF diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 3252caa..dcd2bb5 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -1,4 +1,86 @@ package agent +import ( + "bufio" + "bytes" + "log/slog" + + "mrvacommander/pkg/common" + "mrvacommander/pkg/queue" + "mrvacommander/pkg/storage" + + "os" + "os/exec" + "path" + "strconv" + "strings" +) + type RunnerSingle struct { + queue queue.Queue +} + +func NewRunnerSingle(numWorkers int, queue queue.Queue) *RunnerSingle { + r := RunnerSingle{queue: queue} + + for id := 1; id <= numWorkers; id++ { + go r.worker(id) + } + return &r +} + +func (r *RunnerSingle) worker(wid int) { + var job common.AnalyzeJob + + for { + job = <-r.queue.Jobs() + + slog.Debug("Picking up job", "job", job, "worker", wid) + + cwd, err := os.Getwd() + if err != nil { + slog.Error("RunJob: cwd problem: ", "error", err) + continue + } + + slog.Debug("Analysis: running", "job", job) + storage.SetStatus(job.QueryPackId, job.ORL, common.StatusQueued) + cmd := exec.Command(path.Join(cwd, "cmd", "run-analysis.sh"), + strconv.FormatInt(int64(job.QueryPackId), 10), + job.QueryLanguage, job.ORL.Owner, job.ORL.Repo) + + out, err := cmd.CombinedOutput() + if err != nil { + slog.Error("Analysis command failed: exit code: ", "error", err, "job", job) + slog.Error("Analysis command failed: ", "job", job, "output", out) + storage.SetStatus(job.QueryPackId, job.ORL, common.StatusError) + continue + } + slog.Debug("Analysis run finished", "job", job) + + // Get the SARIF ouput location + sr := bufio.NewScanner(bytes.NewReader(out)) + sr.Split(bufio.ScanLines) + for { + more := sr.Scan() + if !more { + slog.Error("Analysis run failed to report result: ", "output", out) + break + } + fields := strings.Fields(sr.Text()) + if len(fields) >= 3 { + if fields[0] == "run-analysis-output" { + slog.Debug("Analysis run successful: ", "job", job, "location", fields[2]) + res := common.AnalyzeResult{ + RunAnalysisSARIF: fields[2], // Abs. path from run-analysis.sh + RunAnalysisBQRS: "", // FIXME? see note in run-analysis.sh + } + r.queue.Results() <- res + storage.SetStatus(job.QueryPackId, job.ORL, common.StatusSuccess) + storage.SetResult(job.QueryPackId, job.ORL, res) + break + } + } + } + } } diff --git a/pkg/logger/types.go b/pkg/logger/types.go index 66d4be0..51afee8 100644 --- a/pkg/logger/types.go +++ b/pkg/logger/types.go @@ -2,3 +2,8 @@ package logger type LoggerSingle struct { } + +func NewLoggerSingle() *LoggerSingle { + l := LoggerSingle{} + return &l +} diff --git a/pkg/queue/interfaces.go b/pkg/queue/interfaces.go index df566c7..ed9c02e 100644 --- a/pkg/queue/interfaces.go +++ b/pkg/queue/interfaces.go @@ -1,4 +1,14 @@ package queue +import ( + "mrvacommander/pkg/common" + "mrvacommander/pkg/storage" +) + type Queue interface { + Jobs() chan common.AnalyzeJob + Results() chan common.AnalyzeResult + StartAnalyses(analysis_repos *map[common.OwnerRepo]storage.DBLocation, + session_id int, + session_language string) } diff --git a/pkg/queue/queue.go b/pkg/queue/queue.go index 4de83b8..d115a49 100644 --- a/pkg/queue/queue.go +++ b/pkg/queue/queue.go @@ -6,13 +6,15 @@ import ( "mrvacommander/pkg/storage" ) -var ( - NumWorkers int - Jobs chan common.AnalyzeJob - Results chan common.AnalyzeResult -) +func (q *QueueSingle) Jobs() chan common.AnalyzeJob { + return q.jobs +} -func StartAnalyses(analysis_repos *map[common.OwnerRepo]storage.DBLocation, session_id int, +func (q *QueueSingle) Results() chan common.AnalyzeResult { + return q.results +} + +func (q *QueueSingle) StartAnalyses(analysis_repos *map[common.OwnerRepo]storage.DBLocation, session_id int, session_language string) { slog.Debug("Queueing codeql database analyze jobs") @@ -23,7 +25,7 @@ func StartAnalyses(analysis_repos *map[common.OwnerRepo]storage.DBLocation, sess ORL: orl, } - Jobs <- info + q.jobs <- info storage.SetStatus(session_id, orl, common.StatusQueued) storage.AddJob(session_id, info) } diff --git a/pkg/queue/types.go b/pkg/queue/types.go index 33f5204..ea39e74 100644 --- a/pkg/queue/types.go +++ b/pkg/queue/types.go @@ -1,4 +1,17 @@ package queue +import "mrvacommander/pkg/common" + type QueueSingle struct { + NumWorkers int + jobs chan common.AnalyzeJob + results chan common.AnalyzeResult +} + +func NewQueueSingle(numWorkers int) *QueueSingle { + q := QueueSingle{} + q.jobs = make(chan common.AnalyzeJob, 10) + q.results = make(chan common.AnalyzeResult, 10) + q.NumWorkers = numWorkers + return &q } diff --git a/pkg/server/server.go b/pkg/server/server.go index e47c721..816a657 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -16,15 +16,11 @@ import ( "time" "mrvacommander/pkg/common" - "mrvacommander/pkg/queue" "mrvacommander/pkg/storage" "github.com/gorilla/mux" ) -func (c *CommanderSingle) Run() { -} - func (c *CommanderSingle) Setup(st *State) { r := mux.NewRouter() c.st = st @@ -289,7 +285,7 @@ func (c *CommanderSingle) MirvaRequest(w http.ResponseWriter, r *http.Request) { not_found_repos, analysisRepos := c.st.Storage.FindAvailableDBs(session_repositories) - queue.StartAnalyses(analysisRepos, session_id, session_language) + c.queue.StartAnalyses(analysisRepos, session_id, session_language) si := SessionInfo{ ID: session_id, diff --git a/pkg/server/types.go b/pkg/server/types.go index d7b3ac8..6d875c2 100644 --- a/pkg/server/types.go +++ b/pkg/server/types.go @@ -26,7 +26,13 @@ type SessionInfo struct { } type CommanderSingle struct { - st *State + st *State + queue queue.Queue +} + +func NewCommanderSingle(s *State, q queue.Queue) *CommanderSingle { + c := CommanderSingle{s, q} + return &c } type State struct { diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 436170c..d2cc2f4 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -27,6 +27,11 @@ type StorageSingle struct { CurrentID int } +func NewStorageSingle(startingID int) *StorageSingle { + s := StorageSingle{CurrentID: startingID} + return &s +} + func (s *StorageSingle) NextID() int { s.CurrentID += 1 return s.CurrentID @@ -121,6 +126,12 @@ func GetResult(js common.JobSpec) common.AnalyzeResult { return ar } +func SetResult(sessionid int, orl common.OwnerRepo, ar common.AnalyzeResult) { + mutex.Lock() + defer mutex.Unlock() + result[common.JobSpec{ID: sessionid, OwnerRepo: orl}] = ar +} + func PackageResults(ar common.AnalyzeResult, owre common.OwnerRepo, vaid int) (zipPath string, e error) { slog.Debug("Readying zip file with .sarif/.bqrs", "analyze-result", ar)