wip: start container version of server

README.md

@@ -6,7 +6,9 @@ TODO Style notes
 - NO package init() functions
 - Dynamic behaviour must be explicit
  
-## cross-compile server on host, run it in container 
+## Cross-compile server on host, run it in container 
+These are simple steps using a single container.
+
 1.  build server on host
 
         GOOS=linux GOARCH=arm64 go build
@@ -27,21 +29,42 @@ TODO Style notes
 
         cd /mrva/mrvacommander/cmd/server/ && ./server
 
-## Using docker
+## Using docker-compose
-1.  start the services
+Steps to build and run the server in a multi-container environment set up by docker-compose.
 
+1.  Build server on host
+
+        cd ~/work-gh/mrva/mrvacommander/cmd/server/
+        GOOS=linux GOARCH=arm64 go build
+
+1.  Start the containers
+
+        cd ~/work-gh/mrva/mrvacommander/
         docker-compose up -d
-
     
-2.  get status
+4.  Run server in its container
+
+        cd ~/work-gh/mrva/mrvacommander/
+        docker exec -it server bash
+        cd /mrva/mrvacommander/cmd/server/ 
+        ./server -loglevel=debug -mode=container
+
+1.  Test server via remote client by following the steps in [gh-mrva](https://github.com/hohn/gh-mrva/blob/connection-redirect/README.org#compacted-edit-run-debug-cycle)
+    
+
+
+
+Some general docker-compose commands
+
+2.  Get service status
 
         docker-compose ps
         
-3.  stop services
+3.  Stop services
 
         docker-compose down
         
-4.  view all logs
+4.  View all logs
 
         docker-compose logs
 
@@ -50,8 +73,11 @@ TODO Style notes
         docker exec -it server bash
         curl -I postgres:5432
         curl -I http://rabbitmq:15672
-        
+
-1.  Accessing PostgreSQL
+
+Some postgres specific commands
+
+1.  Access PostgreSQL
     
         psql -h localhost -p 5432 -U exampleuser -d exampledb
 
@@ -59,6 +85,8 @@ TODO Style notes
     
         \dt
 
+To run pgmin, the minimal go/postgres test part of this repository:
+
 1.  Run pgmin
 
     ```sh

cmd/server/main.go

@@ -85,7 +85,28 @@ func main() {
 		sc.Setup(&state) // sc is part of state and dereferences it
 
 	case "container":
-		// Assemble cccontainer
+		// Assemble container version
+		sq := queue.NewQueueSingle(2) // FIXME take value from configuration
+		sc := server.NewCommanderSingle(nil, sq)
+		sl := logger.NewLoggerSingle()
+		ss, err := storage.NewStorageContainer(config.Storage.StartingID)
+		if err != nil {
+			slog.Error("Unable to initialize storage")
+			os.Exit(1)
+		}
+
+		sr := agent.NewRunnerSingle(2, sq) // FIXME take value from configuration
+
+		state := server.State{
+			Commander: sc,
+			Logger:    sl,
+			Queue:     sq,
+			Storage:   ss,
+			Runner:    sr,
+		}
+
+		sc.Setup(&state) // sc is part of state and dereferences it
+
 	case "cluster":
 		// Assemble cccluster
 	default:

pkg/server/server.go

@@ -54,6 +54,7 @@ func (c *CommanderSingle) Setup(st *State) {
 	//
 	// Bind to a port and pass our router in
 	//
+	// TODO make this a configuration entry
 	log.Fatal(http.ListenAndServe(":8080", r))
 }
 

pkg/storage/container.go

@@ -0,0 +1,314 @@
+package storage
+
+import (
+	"fmt"
+	"log/slog"
+	"mrvacommander/pkg/common"
+	"sync"
+
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+)
+
+var (
+	DBmutex sync.Mutex
+)
+
+func (s *StorageContainer) NextID() int {
+	// TODO update via db
+	return 12345
+}
+
+func (s *StorageContainer) SaveQueryPack(tgz []byte, sessionID int) (storagePath string, error error) {
+	// TODO save and return path
+	return "todo:no-path-yet", nil
+}
+
+func (s *StorageContainer) FindAvailableDBs(analysisReposRequested []common.OwnerRepo) (notFoundRepos []common.OwnerRepo, analysisRepos *map[common.OwnerRepo]DBLocation) {
+	// TODO  s.FindAvailableDBs() via postgres
+	analysisRepos = &map[common.OwnerRepo]DBLocation{}
+	notFoundRepos = []common.OwnerRepo{}
+
+	return notFoundRepos, analysisRepos
+}
+
+func NewStorageContainer(startingID int) (*StorageContainer, error) {
+	// Set up the database connection string
+	const (
+		host     = "postgres"
+		port     = 5432
+		user     = "exampleuser"
+		password = "examplepass"
+		dbname   = "exampledb"
+	)
+
+	// Open the database connection
+	dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
+		host, port, user, password, dbname)
+	db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{})
+	if err != nil {
+		slog.Error("Error connecting to the database", "err", err)
+		return nil, err
+	}
+
+	// Check and set up the database
+	s := StorageContainer{RequestID: startingID, DB: db}
+	if s.hasTables() {
+		s.loadState()
+	} else {
+		if err = s.setupDB(); err != nil {
+			return nil, err
+		}
+		s.setFresh()
+	}
+
+	return &s, nil
+}
+
+func (s *StorageContainer) setFresh() {
+	// TODO Set initial state
+}
+
+func (s *StorageContainer) setupDB() error {
+	// TODO Migrate the schemas
+	msg := "Failed to initialize database "
+
+	if err := s.DB.AutoMigrate(&DBInfo{}); err != nil {
+		slog.Error(msg, "table", "dbinfo")
+		return err
+	}
+	if err := s.DB.AutoMigrate(&DBJobs{}); err != nil {
+		slog.Error(msg, "table", "dbjobs")
+		return err
+	}
+	if err := s.DB.AutoMigrate(&DBResult{}); err != nil {
+		slog.Error(msg, "table", "dbresult")
+		return err
+	}
+	if err := s.DB.AutoMigrate(&DBStatus{}); err != nil {
+		slog.Error(msg, "table", "dbstatus")
+		return err
+	}
+
+	return nil
+}
+
+func (s *StorageContainer) loadState() {
+	// TODO load the state
+	return
+}
+
+func (s *StorageContainer) hasTables() bool {
+	// TODO sql query to check for tables
+	return false
+}
+
+// ================ TODO migrate
+
+// func (s *StorageSingle) NextID() int {
+// 	s.RequestID += 1
+// 	return s.RequestID
+// }
+
+// func (s *StorageSingle) SaveQueryPack(tgz []byte, sessionId int) (string, error) {
+// 	// Save the tar.gz body
+// 	cwd, err := os.Getwd()
+// 	if err != nil {
+// 		slog.Error("No working directory")
+// 		panic(err)
+// 	}
+
+// 	dirpath := path.Join(cwd, "var", "codeql", "querypacks")
+// 	if err := os.MkdirAll(dirpath, 0755); err != nil {
+// 		slog.Error("Unable to create query pack output directory",
+// 			"dir", dirpath)
+// 		return "", err
+// 	}
+
+// 	fpath := path.Join(dirpath, fmt.Sprintf("qp-%d.tgz", sessionId))
+// 	err = os.WriteFile(fpath, tgz, 0644)
+// 	if err != nil {
+// 		slog.Error("unable to save querypack body decoding error", "path", fpath)
+// 		return "", err
+// 	} else {
+// 		slog.Info("Query pack saved to ", "path", fpath)
+// 	}
+
+// 	return fpath, nil
+// }
+
+// //		Determine for which repositories codeql databases are available.
+// //
+// //	 Those will be the analysis_repos.  The rest will be skipped.
+// func (s *StorageSingle) FindAvailableDBs(analysisReposRequested []common.OwnerRepo) (notFoundRepos []common.OwnerRepo,
+// 	analysisRepos *map[common.OwnerRepo]DBLocation) {
+// 	slog.Debug("Looking for available CodeQL databases")
+
+// 	cwd, err := os.Getwd()
+// 	if err != nil {
+// 		slog.Error("No working directory")
+// 		return
+// 	}
+
+// 	analysisRepos = &map[common.OwnerRepo]DBLocation{}
+
+// 	notFoundRepos = []common.OwnerRepo{}
+
+// 	for _, rep := range analysisReposRequested {
+// 		dbPrefix := filepath.Join(cwd, "codeql", "dbs", rep.Owner, rep.Repo)
+// 		dbName := fmt.Sprintf("%s_%s_db.zip", rep.Owner, rep.Repo)
+// 		dbPath := filepath.Join(dbPrefix, dbName)
+
+// 		if _, err := os.Stat(dbPath); errors.Is(err, fs.ErrNotExist) {
+// 			slog.Info("Database does not exist for repository ", "owner/repo", rep,
+// 				"path", dbPath)
+// 			notFoundRepos = append(notFoundRepos, rep)
+// 		} else {
+// 			slog.Info("Found database for ", "owner/repo", rep, "path", dbPath)
+// 			(*analysisRepos)[rep] = DBLocation{Prefix: dbPrefix, File: dbName}
+// 		}
+// 	}
+// 	return notFoundRepos, analysisRepos
+// }
+
+// func ArtifactURL(js common.JobSpec, vaid int) (string, error) {
+// 	// We're looking for paths like
+// 	// codeql/sarif/google/flatbuffers/google_flatbuffers.sarif
+
+// 	ar := GetResult(js)
+
+// 	hostname, err := os.Hostname()
+// 	if err != nil {
+// 		slog.Error("No host name found")
+// 		return "", nil
+// 	}
+
+// 	zfpath, err := PackageResults(ar, js.OwnerRepo, vaid)
+// 	if err != nil {
+// 		slog.Error("Error packaging results:", "error", err)
+// 		return "", err
+// 	}
+// 	au := fmt.Sprintf("http://%s:8080/download-server/%s", hostname, zfpath)
+// 	return au, nil
+// }
+
+// func GetResult(js common.JobSpec) common.AnalyzeResult {
+// 	mutex.Lock()
+// 	defer mutex.Unlock()
+// 	ar := result[js]
+// 	return ar
+// }
+
+// func SetResult(sessionid int, orl common.OwnerRepo, ar common.AnalyzeResult) {
+// 	mutex.Lock()
+// 	defer mutex.Unlock()
+// 	result[common.JobSpec{RequestID: sessionid, OwnerRepo: orl}] = ar
+// }
+
+// func PackageResults(ar common.AnalyzeResult, owre common.OwnerRepo, vaid int) (zipPath string, e error) {
+// 	slog.Debug("Readying zip file with .sarif/.bqrs", "analyze-result", ar)
+
+// 	cwd, err := os.Getwd()
+// 	if err != nil {
+// 		slog.Error("No working directory")
+// 		panic(err)
+// 	}
+
+// 	// Ensure the output directory exists
+// 	dirpath := path.Join(cwd, "var", "codeql", "localrun", "results")
+// 	if err := os.MkdirAll(dirpath, 0755); err != nil {
+// 		slog.Error("Unable to create results output directory",
+// 			"dir", dirpath)
+// 		return "", err
+// 	}
+
+// 	// Create a new zip file
+// 	zpath := path.Join(dirpath, fmt.Sprintf("results-%s-%s-%d.zip", owre.Owner, owre.Repo, vaid))
+
+// 	zfile, err := os.Create(zpath)
+// 	if err != nil {
+// 		return "", err
+// 	}
+// 	defer zfile.Close()
+
+// 	// Create a new zip writer
+// 	zwriter := zip.NewWriter(zfile)
+// 	defer zwriter.Close()
+
+// 	// Add each result file to the zip archive
+// 	names := []([]string){{ar.RunAnalysisSARIF, "results.sarif"}}
+// 	for _, fpath := range names {
+// 		file, err := os.Open(fpath[0])
+// 		if err != nil {
+// 			return "", err
+// 		}
+// 		defer file.Close()
+
+// 		// Create a new file in the zip archive with custom name
+// 		// The client is very specific:
+// 		// if zf.Name != "results.sarif" && zf.Name != "results.bqrs" { continue }
+
+// 		zipEntry, err := zwriter.Create(fpath[1])
+// 		if err != nil {
+// 			return "", err
+// 		}
+
+// 		// Copy the contents of the file to the zip entry
+// 		_, err = io.Copy(zipEntry, file)
+// 		if err != nil {
+// 			return "", err
+// 		}
+// 	}
+// 	return zpath, nil
+// }
+
+// func GetJobList(sessionid int) []common.AnalyzeJob {
+// 	mutex.Lock()
+// 	defer mutex.Unlock()
+// 	return jobs[sessionid]
+// }
+
+// func GetJobInfo(js common.JobSpec) common.JobInfo {
+// 	mutex.Lock()
+// 	defer mutex.Unlock()
+// 	return info[js]
+// }
+
+// func SetJobInfo(js common.JobSpec, ji common.JobInfo) {
+// 	mutex.Lock()
+// 	defer mutex.Unlock()
+// 	info[js] = ji
+// }
+
+// func GetStatus(sessionid int, orl common.OwnerRepo) common.Status {
+// 	mutex.Lock()
+// 	defer mutex.Unlock()
+// 	return status[common.JobSpec{RequestID: sessionid, OwnerRepo: orl}]
+// }
+
+// func ResultAsFile(path string) (string, []byte, error) {
+// 	fpath := path
+// 	if !filepath.IsAbs(path) {
+// 		fpath = "/" + path
+// 	}
+
+// 	file, err := os.ReadFile(fpath)
+// 	if err != nil {
+// 		slog.Warn("Failed to read results file", fpath, err)
+// 		return "", nil, err
+// 	}
+
+// 	return fpath, file, nil
+// }
+
+// func SetStatus(sessionid int, orl common.OwnerRepo, s common.Status) {
+// 	mutex.Lock()
+// 	defer mutex.Unlock()
+// 	status[common.JobSpec{RequestID: sessionid, OwnerRepo: orl}] = s
+// }
+
+// func AddJob(sessionid int, job common.AnalyzeJob) {
+// 	mutex.Lock()
+// 	defer mutex.Unlock()
+// 	jobs[sessionid] = append(jobs[sessionid], job)
+// }

pkg/storage/interfaces.go

@@ -1 +1,10 @@
 package storage
+
+import "mrvacommander/pkg/common"
+
+type Storage interface {
+	NextID() int
+	SaveQueryPack(tgz []byte, sessionID int) (storagePath string, error error)
+	FindAvailableDBs(analysisReposRequested []common.OwnerRepo) (not_found_repos []common.OwnerRepo,
+		analysisRepos *map[common.OwnerRepo]DBLocation)
+}

pkg/storage/storage.go

@@ -23,18 +23,14 @@ var (
 	mutex  sync.Mutex
 )
 
-type StorageSingle struct {
-	CurrentID int
-}
-
 func NewStorageSingle(startingID int) *StorageSingle {
-	s := StorageSingle{CurrentID: startingID}
+	s := StorageSingle{currentID: startingID}
 	return &s
 }
 
 func (s *StorageSingle) NextID() int {
-	s.CurrentID += 1
+	s.currentID += 1
-	return s.CurrentID
+	return s.currentID
 }
 
 func (s *StorageSingle) SaveQueryPack(tgz []byte, sessionId int) (string, error) {

pkg/storage/types.go

@@ -2,16 +2,53 @@ package storage
 
 import (
 	"mrvacommander/pkg/common"
-)
 
-type Storage interface {
+	"gorm.io/gorm"
-	NextID() int
+)
-	SaveQueryPack(tgz []byte, sessionID int) (storagePath string, error error)
-	FindAvailableDBs(analysisReposRequested []common.OwnerRepo) (not_found_repos []common.OwnerRepo,
-		analysisRepos *map[common.OwnerRepo]DBLocation)
-}
 
 type DBLocation struct {
 	Prefix string
 	File   string
 }
+
+type StorageSingle struct {
+	currentID int
+}
+
+type DBInfo struct {
+	// Database version of
+	// info map[common.JobSpec]common.JobInfo       = make(map[common.JobSpec]common.JobInfo)
+	gorm.Model
+	Key     common.JobSpec
+	JobInfo common.JobInfo
+}
+
+type DBJobs struct {
+	// Database version of
+	// jobs   map[int][]common.AnalyzeJob             = make(map[int][]common.AnalyzeJob)
+	gorm.Model
+	Key        int
+	AnalyzeJob common.AnalyzeJob
+}
+
+type DBResult struct {
+	// Database version of
+	// result map[common.JobSpec]common.AnalyzeResult = make(map[common.JobSpec]common.AnalyzeResult)
+	gorm.Model
+	Key           common.JobSpec
+	AnalyzeResult common.AnalyzeResult
+}
+
+type DBStatus struct {
+	// Database version of
+	// status map[common.JobSpec]common.Status        = make(map[common.JobSpec]common.Status)
+	gorm.Model
+	Key    common.JobSpec
+	Status common.Status
+}
+
+type StorageContainer struct {
+	// Database version of StorageSingle
+	RequestID int
+	DB        *gorm.DB
+}