wip: convert run-analysis.sh to golang version

README.md

@@ -0,0 +1,7 @@
+# Overview
+
+TODO diagram
+
+TODO note: NO package init() functions
+Dynamic behaviour must be explicit
+ 

cmd/agent/main.go

@@ -0,0 +1 @@
+package agent

cmd/server/bin/run-analysis.sh

@@ -1,58 +0,0 @@
-#!/bin/bash -x -e
-#*    Minimal setup to run analysis using information provided by a request
-
-#*    Take output saved by the server
-QUERYPACKID=$1
-shift
-QUERYLANGUAGE=$1
-shift
-
-# and
-DBOWNER=$1
-shift
-DBREPO=$1
-
-# FIXME Provide this via environment or explicit argument
-GMSROOT=/Users/hohn/local/ghes-mirva-server
-GMSROOT=/Users/hohn/work-gh/mrva/mrvacommander/cmd/server
-
-#* Set up derived paths
-DBPATH=$GMSROOT/var/codeql/dbs/$DBOWNER/$DBREPO
-DBZIP=$GMSROOT/codeql/dbs/$DBOWNER/$DBREPO/${DBOWNER}_${DBREPO}_db.zip
-DBEXTRACT=$GMSROOT/var/codeql/dbs/$DBOWNER/$DBREPO
-
-QUERYPACK=$GMSROOT/var/codeql/querypacks/qp-$QUERYPACKID.tgz
-QUERYEXTRACT=$GMSROOT/var/codeql/querypacks/qp-$QUERYPACKID
-
-QUERYOUTD=$GMSROOT/var/codeql/sarif/localrun/$DBOWNER/$DBREPO
-QUERYOUTF=$QUERYOUTD/${DBOWNER}_${DBREPO}.sarif
-
-#*    Prep work before running the command
-
-#**        Extract database
-mkdir -p  $DBEXTRACT && cd $DBEXTRACT
-unzip -o -q $DBZIP
-
-#   Extract query pack
-mkdir -p $QUERYEXTRACT && cd $QUERYEXTRACT
-tar zxf $QUERYPACK
-
-#**        Prepare target directory
-mkdir -p $QUERYOUTD
-
-#*    run database analyze
-cd $GMSROOT
-codeql database analyze --format=sarif-latest --rerun   \
-       --output $QUERYOUTF                              \
-       -j8                                              \
-       -- $DBPATH $QUERYEXTRACT 
-
-#*    report result
-printf "run-analysis-output in %s\n" $QUERYOUTF
-
-# TODO Is the bqrs really necessary?  It can also be found by the go code at this point. 
-# TODO The name of the query is only found in the query pack itself and would have
-# to be extracted from there.
-
-# BQRSPATH=$GMSROOT/var/codeql/dbs/$DBOWNER/$DBREPO/$LANGUAGE/results/codeql-remote/query/
-# printf "run-analysis-bqrs in %s\n" $QUERYOUTF

pkg/agent/agent.go

@@ -1,19 +1,21 @@
 package agent
 
 import (
-	"bufio"
-	"bytes"
-	"log/slog"
-
 	"mrvacommander/pkg/common"
 	"mrvacommander/pkg/queue"
 	"mrvacommander/pkg/storage"
 
+	"log/slog"
+
+	"archive/tar"
+	"archive/zip"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"path/filepath"
+
 	"os"
 	"os/exec"
-	"path"
-	"strconv"
-	"strings"
 )
 
 type RunnerSingle struct {
@@ -37,50 +39,196 @@ func (r *RunnerSingle) worker(wid int) {
 
 		slog.Debug("Picking up job", "job", job, "worker", wid)
 
-		cwd, err := os.Getwd()
-		if err != nil {
-			slog.Error("RunJob: cwd problem: ", "error", err)
-			continue
-		}
-
 		slog.Debug("Analysis: running", "job", job)
 		storage.SetStatus(job.QueryPackId, job.ORL, common.StatusQueued)
-		cmd := exec.Command(path.Join(cwd, "bin", "run-analysis.sh"),
-			strconv.FormatInt(int64(job.QueryPackId), 10),
-			job.QueryLanguage, job.ORL.Owner, job.ORL.Repo)
 
-		out, err := cmd.CombinedOutput()
+		resultFile, err := r.RunAnalysis(job)
 		if err != nil {
-			slog.Error("Analysis command failed: exit code: ", "error", err, "job", job)
-			slog.Error("Analysis command failed: ", "job", job, "output", out)
-			storage.SetStatus(job.QueryPackId, job.ORL, common.StatusError)
 			continue
 		}
+
 		slog.Debug("Analysis run finished", "job", job)
 
-		// Get the SARIF ouput location
+		res := common.AnalyzeResult{
-		sr := bufio.NewScanner(bytes.NewReader(out))
+			RunAnalysisSARIF: resultFile,
-		sr.Split(bufio.ScanLines)
+			RunAnalysisBQRS:  "", // FIXME ?
-		for {
-			more := sr.Scan()
-			if !more {
-				slog.Error("Analysis run failed to report result: ", "output", out)
-				break
-			}
-			fields := strings.Fields(sr.Text())
-			if len(fields) >= 3 {
-				if fields[0] == "run-analysis-output" {
-					slog.Debug("Analysis run successful: ", "job", job, "location", fields[2])
-					res := common.AnalyzeResult{
-						RunAnalysisSARIF: fields[2], // Abs. path from run-analysis.sh
-						RunAnalysisBQRS:  "",        // FIXME? see note in run-analysis.sh
-					}
-					r.queue.Results() <- res
-					storage.SetStatus(job.QueryPackId, job.ORL, common.StatusSuccess)
-					storage.SetResult(job.QueryPackId, job.ORL, res)
-					break
-				}
-			}
 		}
+		r.queue.Results() <- res
+		storage.SetStatus(job.QueryPackId, job.ORL, common.StatusSuccess)
+		storage.SetResult(job.QueryPackId, job.ORL, res)
+
 	}
 }
+
+func (r *RunnerSingle) RunAnalysis(job common.AnalyzeJob) (string, error) {
+	// TODO Add multi-language tests including queryLanguage
+	// queryPackID, queryLanguage, dbOwner, dbRepo :=
+	// 	job.QueryPackId, job.QueryLanguage, job.ORL.Owner, job.ORL.Repo
+	queryPackID, dbOwner, dbRepo :=
+		job.QueryPackId, job.ORL.Owner, job.ORL.Repo
+
+	// FIXME Provide this via environment or explicit argument
+	gmsRoot := "/Users/hohn/work-gh/mrva/mrvacommander/cmd/server"
+
+	// Set up derived paths
+	dbPath := filepath.Join(gmsRoot, "var/codeql/dbs", dbOwner, dbRepo)
+	dbZip := filepath.Join(gmsRoot, "codeql/dbs", dbOwner, dbRepo,
+		fmt.Sprintf("%s_%s_db.zip", dbOwner, dbRepo))
+	dbExtract := filepath.Join(gmsRoot, "var/codeql/dbs", dbOwner, dbRepo)
+
+	queryPack := filepath.Join(gmsRoot,
+		"var/codeql/querypacks", fmt.Sprintf("qp-%d.tgz", queryPackID))
+	queryExtract := filepath.Join(gmsRoot,
+		"var/codeql/querypacks", fmt.Sprintf("qp-%d", queryPackID))
+
+	queryOutDir := filepath.Join(gmsRoot,
+		"var/codeql/sarif/localrun", dbOwner, dbRepo)
+	queryOutFile := filepath.Join(queryOutDir,
+		fmt.Sprintf("%s_%s.sarif", dbOwner, dbRepo))
+
+	// Prepare directory, extract database
+	if err := os.MkdirAll(dbExtract, 0755); err != nil {
+		slog.Error("Failed to create DB directory %s: %v", dbExtract, err)
+		return "", err
+	}
+
+	if err := unzipFile(dbZip, dbExtract); err != nil {
+		slog.Error("Failed to unzip DB %s: %v", dbZip, err)
+		return "", err
+	}
+
+	// Prepare directory, extract query pack
+	if err := os.MkdirAll(queryExtract, 0755); err != nil {
+		slog.Error("Failed to create query pack directory %s: %v", queryExtract, err)
+		return "", err
+	}
+
+	if err := untarGz(queryPack, queryExtract); err != nil {
+		slog.Error("Failed to extract querypack %s: %v", queryPack, err)
+		return "", err
+	}
+
+	// Prepare query result directory
+	if err := os.MkdirAll(queryOutDir, 0755); err != nil {
+		slog.Error("Failed to create query result directory %s: %v", queryOutDir, err)
+		return "", err
+	}
+
+	// Run database analyze
+	cmd := exec.Command("codeql", "database", "analyze",
+		"--format=sarif-latest", "--rerun", "--output", queryOutFile,
+		"-j8", dbPath, queryExtract)
+	cmd.Dir = gmsRoot
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	if err := cmd.Run(); err != nil {
+		slog.Error("codeql database analyze failed:", "error", err, "job", job)
+		storage.SetStatus(job.QueryPackId, job.ORL, common.StatusError)
+		return "", err
+	}
+
+	// Return result path
+	return queryOutFile, nil
+}
+
+// unzipFile extracts a zip file to the specified destination
+func unzipFile(zipFile, dest string) error {
+	r, err := zip.OpenReader(zipFile)
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+
+	for _, f := range r.File {
+		fPath := filepath.Join(dest, f.Name)
+		if f.FileInfo().IsDir() {
+			if err := os.MkdirAll(fPath, os.ModePerm); err != nil {
+				return err
+			}
+			continue
+		}
+
+		if err := os.MkdirAll(filepath.Dir(fPath), os.ModePerm); err != nil {
+			return err
+		}
+
+		outFile, err := os.OpenFile(fPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
+		if err != nil {
+			return err
+		}
+
+		rc, err := f.Open()
+		if err != nil {
+			outFile.Close()
+			return err
+		}
+
+		_, err = io.Copy(outFile, rc)
+
+		outFile.Close()
+		rc.Close()
+
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// untarGz extracts a tar.gz file to the specified destination.
+func untarGz(tarGzFile, dest string) error {
+	file, err := os.Open(tarGzFile)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	gzr, err := gzip.NewReader(file)
+	if err != nil {
+		return err
+	}
+	defer gzr.Close()
+
+	return untar(gzr, dest)
+}
+
+// untar extracts a tar archive to the specified destination.
+func untar(r io.Reader, dest string) error {
+	tr := tar.NewReader(r)
+
+	for {
+		header, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+
+		fPath := filepath.Join(dest, header.Name)
+		if header.Typeflag == tar.TypeDir {
+			if err := os.MkdirAll(fPath, os.ModePerm); err != nil {
+				return err
+			}
+		} else {
+			if err := os.MkdirAll(filepath.Dir(fPath), os.ModePerm); err != nil {
+				return err
+			}
+
+			outFile, err := os.OpenFile(fPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(header.Mode))
+			if err != nil {
+				return err
+			}
+
+			if _, err := io.Copy(outFile, tr); err != nil {
+				outFile.Close()
+				return err
+			}
+
+			outFile.Close()
+		}
+	}
+
+	return nil
+}