package agent import ( "mrvacommander/pkg/common" "mrvacommander/pkg/logger" "mrvacommander/pkg/qpstore" "mrvacommander/pkg/queue" "mrvacommander/pkg/storage" "log/slog" "archive/tar" "archive/zip" "compress/gzip" "fmt" "io" "path/filepath" "os" "os/exec" ) type RunnerSingle struct { queue queue.Queue } func NewRunnerSingle(numWorkers int, queue queue.Queue) *RunnerSingle { r := RunnerSingle{queue: queue} for id := 1; id <= numWorkers; id++ { go r.worker(id) } return &r } type Visibles struct { Logger logger.Logger Queue queue.Queue // TODO extra package for query pack storage QueryPackStore qpstore.Storage // TODO extra package for ql db storage QLDBStore storage.Storage } func (c *RunnerSingle) Setup(st *Visibles) { return } func (r *RunnerSingle) worker(wid int) { var job common.AnalyzeJob for { job = <-r.queue.Jobs() slog.Debug("Picking up job", "job", job, "worker", wid) slog.Debug("Analysis: running", "job", job) storage.SetStatus(job.QueryPackId, job.ORepo, common.StatusQueued) resultFile, err := r.RunAnalysis(job) if err != nil { continue } slog.Debug("Analysis run finished", "job", job) res := common.AnalyzeResult{ RunAnalysisSARIF: resultFile, RunAnalysisBQRS: "", // FIXME ? } r.queue.Results() <- res storage.SetStatus(job.QueryPackId, job.ORepo, common.StatusSuccess) storage.SetResult(job.QueryPackId, job.ORepo, res) } } func (r *RunnerSingle) RunAnalysis(job common.AnalyzeJob) (string, error) { // TODO Add multi-language tests including queryLanguage // queryPackID, queryLanguage, dbOwner, dbRepo := // job.QueryPackId, job.QueryLanguage, job.ORL.Owner, job.ORL.Repo queryPackID, dbOwner, dbRepo := job.QueryPackId, job.ORepo.Owner, job.ORepo.Repo serverRoot := os.Getenv("MRVA_SERVER_ROOT") // Set up derived paths dbPath := filepath.Join(serverRoot, "var/codeql/dbs", dbOwner, dbRepo) dbZip := filepath.Join(serverRoot, "codeql/dbs", dbOwner, dbRepo, fmt.Sprintf("%s_%s_db.zip", dbOwner, dbRepo)) dbExtract := filepath.Join(serverRoot, "var/codeql/dbs", dbOwner, dbRepo) queryPack := filepath.Join(serverRoot, "var/codeql/querypacks", fmt.Sprintf("qp-%d.tgz", queryPackID)) queryExtract := filepath.Join(serverRoot, "var/codeql/querypacks", fmt.Sprintf("qp-%d", queryPackID)) queryOutDir := filepath.Join(serverRoot, "var/codeql/sarif/localrun", dbOwner, dbRepo) queryOutFile := filepath.Join(queryOutDir, fmt.Sprintf("%s_%s.sarif", dbOwner, dbRepo)) // Prepare directory, extract database if err := os.MkdirAll(dbExtract, 0755); err != nil { slog.Error("Failed to create DB directory %s: %v", dbExtract, err) return "", err } if err := unzipFile(dbZip, dbExtract); err != nil { slog.Error("Failed to unzip DB", dbZip, err) return "", err } // Prepare directory, extract query pack if err := os.MkdirAll(queryExtract, 0755); err != nil { slog.Error("Failed to create query pack directory %s: %v", queryExtract, err) return "", err } if err := untarGz(queryPack, queryExtract); err != nil { slog.Error("Failed to extract querypack %s: %v", queryPack, err) return "", err } // Prepare query result directory if err := os.MkdirAll(queryOutDir, 0755); err != nil { slog.Error("Failed to create query result directory %s: %v", queryOutDir, err) return "", err } // Run database analyze cmd := exec.Command("codeql", "database", "analyze", "--format=sarif-latest", "--rerun", "--output", queryOutFile, "-j8", dbPath, queryExtract) cmd.Dir = serverRoot cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { slog.Error("codeql database analyze failed:", "error", err, "job", job) storage.SetStatus(job.QueryPackId, job.ORepo, common.StatusError) return "", err } // Return result path return queryOutFile, nil } // unzipFile extracts a zip file to the specified destination func unzipFile(zipFile, dest string) error { r, err := zip.OpenReader(zipFile) if err != nil { return err } defer r.Close() for _, f := range r.File { fPath := filepath.Join(dest, f.Name) if f.FileInfo().IsDir() { if err := os.MkdirAll(fPath, os.ModePerm); err != nil { return err } continue } if err := os.MkdirAll(filepath.Dir(fPath), os.ModePerm); err != nil { return err } outFile, err := os.OpenFile(fPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) if err != nil { return err } rc, err := f.Open() if err != nil { outFile.Close() return err } _, err = io.Copy(outFile, rc) outFile.Close() rc.Close() if err != nil { return err } } return nil } // untarGz extracts a tar.gz file to the specified destination. func untarGz(tarGzFile, dest string) error { file, err := os.Open(tarGzFile) if err != nil { return err } defer file.Close() gzr, err := gzip.NewReader(file) if err != nil { return err } defer gzr.Close() return untar(gzr, dest) } // untar extracts a tar archive to the specified destination. func untar(r io.Reader, dest string) error { tr := tar.NewReader(r) for { header, err := tr.Next() if err == io.EOF { break } if err != nil { return err } fPath := filepath.Join(dest, header.Name) if header.Typeflag == tar.TypeDir { if err := os.MkdirAll(fPath, os.ModePerm); err != nil { return err } } else { if err := os.MkdirAll(filepath.Dir(fPath), os.ModePerm); err != nil { return err } outFile, err := os.OpenFile(fPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(header.Mode)) if err != nil { return err } if _, err := io.Copy(outFile, tr); err != nil { outFile.Close() return err } outFile.Close() } } return nil }