wip: Make cross-module visibility explicit via Visibles structs

All access is/will be through interfaces accessed through these structs.

This introduces several distinct storage units:
+ DB for server state
+ DB for codeql databases
+ query pack store

The steps for manually creating needed databases are in the README

README.md

@@ -86,6 +86,33 @@ Some postgres specific commands
     
         \dt
 
+1.  Examine a table
+
+        select * from db_infos
+
+1.  Show all columns in a specific table
+
+        \d+ db_infos
+
+1.  Miscellany
+
+        \pset pager off
+        \lo_import FILE [COMMENT]
+
+
+Manually create needed postgres databases
+
+    # on the host
+    psql -h localhost -p 5432 -U exampleuser -d postgres
+
+    # Conditionally create dbs
+    SELECT 'CREATE DATABASE server_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'server_db')\gexec
+    SELECT 'CREATE DATABASE querypack_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'querypack_db')\gexec
+    SELECT 'CREATE DATABASE qldb_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'qldb_db')\gexec
+
+    # List all dbs
+    \l
+
 To run pgmin, the minimal go/postgres test part of this repository:
 
 1.  Run pgmin

cmd/server/main.go

@@ -74,38 +74,89 @@ func main() {
 		ss := storage.NewStorageSingle(config.Storage.StartingID)
 		sr := agent.NewRunnerSingle(2, sq) // FIXME take value from configuration
 
-		state := server.State{
+		qp, err := storage.NewQueryPackStore(config.Storage.StartingID)
-			Commander: sc,
+		if err != nil {
-			Logger:    sl,
+			slog.Error("Unable to initialize query pack storage")
-			Queue:     sq,
+			os.Exit(1)
-			Storage:   ss,
-			Runner:    sr,
 		}
 
-		sc.Setup(&state) // sc is part of state and dereferences it
+		ql, err := storage.NewQLDBStore()
+		if err != nil {
+			slog.Error("Unable to initialize ql database storage")
+			os.Exit(1)
+		}
+
+		sc.Setup(&server.CommanderVisibles{
+			Logger:         sl,
+			Queue:          sq,
+			ServerStore:    ss,
+			QueryPackStore: qp,
+			QLDBStore:      ql,
+		})
+
+		sl.Setup(&logger.LoggerVisibles{})
+
+		sq.Setup(&queue.QueueVisibles{
+			Logger: sl,
+		})
+
+		ss.Setup(&storage.ServerStorageVisibles{})
+
+		sr.Setup(&agent.RunnerVisibles{
+			Logger:         sl,
+			Queue:          sq,
+			QueryPackStore: qp,
+			QLDBStore:      ql,
+		})
 
 	case "container":
 		// Assemble container version
 		sq := queue.NewQueueSingle(2) // FIXME take value from configuration
 		sc := server.NewCommanderSingle(nil, sq)
 		sl := logger.NewLoggerSingle()
+
 		ss, err := storage.NewStorageContainer(config.Storage.StartingID)
 		if err != nil {
-			slog.Error("Unable to initialize storage")
+			slog.Error("Unable to initialize server storage")
+			os.Exit(1)
+		}
+
+		qp, err := storage.NewQueryPackStore(config.Storage.StartingID)
+		if err != nil {
+			slog.Error("Unable to initialize query pack storage")
+			os.Exit(1)
+		}
+
+		ql, err := storage.NewQLDBStore()
+		if err != nil {
+			slog.Error("Unable to initialize ql database storage")
 			os.Exit(1)
 		}
 
 		sr := agent.NewRunnerSingle(2, sq) // FIXME take value from configuration
 
-		state := server.State{
+		sc.Setup(&server.CommanderVisibles{
-			Commander: sc,
+			Logger:         sl,
-			Logger:    sl,
+			Queue:          sq,
-			Queue:     sq,
+			ServerStore:    ss,
-			Storage:   ss,
+			QueryPackStore: qp,
-			Runner:    sr,
+			QLDBStore:      ql,
-		}
+		})
 
-		sc.Setup(&state) // sc is part of state and dereferences it
+		sl.Setup(&logger.LoggerVisibles{})
+
+		sq.Setup(&queue.QueueVisibles{
+			Logger: sl,
+		})
+
+		ss.Setup(&storage.ServerStorageVisibles{})
+
+		sr.Setup(&agent.RunnerVisibles{
+			Logger:         sl,
+			Queue:          sq,
+			QueryPackStore: qp,
+			QLDBStore:      ql,
+		})
 
 	case "cluster":
 		// Assemble cccluster

pkg/agent/agent.go

@@ -2,6 +2,7 @@ package agent
 
 import (
 	"mrvacommander/pkg/common"
+	"mrvacommander/pkg/logger"
 	"mrvacommander/pkg/queue"
 	"mrvacommander/pkg/storage"
 
@@ -31,6 +32,19 @@ func NewRunnerSingle(numWorkers int, queue queue.Queue) *RunnerSingle {
 	return &r
 }
 
+type RunnerVisibles struct {
+	Logger logger.Logger
+	Queue  queue.Queue
+	// TODO extra package for query pack storage
+	QueryPackStore storage.Storage
+	// TODO extra package for ql db storage
+	QLDBStore storage.Storage
+}
+
+func (c *RunnerSingle) Setup(st *RunnerVisibles) {
+	return
+}
+
 func (r *RunnerSingle) worker(wid int) {
 	var job common.AnalyzeJob
 

pkg/logger/types.go

@@ -1,9 +1,16 @@
 package logger
 
 type LoggerSingle struct {
+	modules *LoggerVisibles
 }
 
 func NewLoggerSingle() *LoggerSingle {
 	l := LoggerSingle{}
 	return &l
 }
+
+type LoggerVisibles struct{}
+
+func (l *LoggerSingle) Setup(v *LoggerVisibles) {
+	l.modules = v
+}

pkg/queue/types.go

@@ -1,11 +1,23 @@
 package queue
 
-import "mrvacommander/pkg/common"
+import (
+	"mrvacommander/pkg/common"
+	"mrvacommander/pkg/logger"
+)
 
 type QueueSingle struct {
 	NumWorkers int
 	jobs       chan common.AnalyzeJob
 	results    chan common.AnalyzeResult
+	modules    *QueueVisibles
+}
+
+type QueueVisibles struct {
+	Logger logger.Logger
+}
+
+func (q *QueueSingle) Setup(v *QueueVisibles) {
+	q.modules = v
 }
 
 func NewQueueSingle(numWorkers int) *QueueSingle {

pkg/server/server.go

@@ -21,7 +21,7 @@ import (
 	"github.com/gorilla/mux"
 )
 
-func (c *CommanderSingle) Setup(st *State) {
+func (c *CommanderSingle) Setup(st *CommanderVisibles) {
 	r := mux.NewRouter()
 	c.st = st
 
@@ -275,7 +275,7 @@ func (c *CommanderSingle) MirvaRequest(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	slog.Info("New mrva run ", "owner", vars["owner"], "repo", vars["repo"])
 
-	session_id := c.st.Storage.NextID()
+	session_id := c.st.ServerStore.NextID()
 	session_owner := vars["owner"]
 	session_controller_repo := vars["repo"]
 	slog.Info("new run", "id: ", fmt.Sprint(session_id), session_owner, session_controller_repo)
@@ -284,7 +284,7 @@ func (c *CommanderSingle) MirvaRequest(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	not_found_repos, analysisRepos := c.st.Storage.FindAvailableDBs(session_repositories)
+	not_found_repos, analysisRepos := c.st.ServerStore.FindAvailableDBs(session_repositories)
 
 	c.queue.StartAnalyses(analysisRepos, session_id, session_language)
 
@@ -492,7 +492,7 @@ func (c *CommanderSingle) extract_tgz(qp string, sessionID int) (string, error)
 		return "", err
 	}
 
-	session_query_pack_tgz_filepath, err := c.st.Storage.SaveQueryPack(tgz, sessionID)
+	session_query_pack_tgz_filepath, err := c.st.ServerStore.SaveQueryPack(tgz, sessionID)
 	if err != nil {
 		return "", err
 	}

pkg/server/types.go

@@ -1,7 +1,6 @@
 package server
 
 import (
-	"mrvacommander/pkg/agent"
 	"mrvacommander/pkg/common"
 	"mrvacommander/pkg/logger"
 	"mrvacommander/pkg/queue"
@@ -26,19 +25,30 @@ type SessionInfo struct {
 }
 
 type CommanderSingle struct {
-	st    *State
+	st *CommanderVisibles
+	// TODO remove:
 	queue queue.Queue
 }
 
-func NewCommanderSingle(s *State, q queue.Queue) *CommanderSingle {
+func NewCommanderSingle(s *CommanderVisibles, q queue.Queue) *CommanderSingle {
 	c := CommanderSingle{s, q}
 	return &c
 }
 
-type State struct {
+// type State struct {
-	Commander Commander
+// 	Commander Commander
-	Logger    logger.Logger
+// 	Logger    logger.Logger
-	Queue     queue.Queue
+// 	Queue     queue.Queue
-	Storage   storage.Storage
+// 	Storage   storage.Storage
-	Runner    agent.Runner
+// 	Runner    agent.Runner
+// }
+
+type CommanderVisibles struct {
+	Logger      logger.Logger
+	Queue       queue.Queue
+	ServerStore storage.Storage
+	// TODO extra package for query pack storage
+	QueryPackStore storage.Storage
+	// TODO extra package for ql db storage
+	QLDBStore storage.Storage
 }

pkg/storage/container.go

@@ -32,6 +32,42 @@ func (s *StorageContainer) FindAvailableDBs(analysisReposRequested []common.Owne
 	return notFoundRepos, analysisRepos
 }
 
+func (s *StorageContainer) Setup(v *ServerStorageVisibles) {
+	s.modules = v
+}
+
+func NewQLDBStore() (*StorageContainer, error) {
+	// TODO set up qldb_db
+	return nil, nil
+}
+
+func NewQueryPackStore(startingID int) (*StorageContainer, error) {
+	// TODO set up querypack_db
+	// TODO drop the startingID
+
+	db, err := ConnectDB(DBSpec{
+		Host:     "postgres",
+		Port:     5432,
+		User:     "exampleuser",
+		Password: "examplepass",
+		DBname:   "querypack_db",
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	s := StorageContainer{RequestID: startingID, DB: db}
+	if err := s.SetupDB(); err != nil {
+		return nil, err
+	}
+
+	if err = s.loadState(); err != nil {
+		return nil, err
+	}
+
+	return &s, nil
+}
+
 func NewStorageContainer(startingID int) (*StorageContainer, error) {
 
 	db, err := ConnectDB(DBSpec{
@@ -39,32 +75,21 @@ func NewStorageContainer(startingID int) (*StorageContainer, error) {
 		Port:     5432,
 		User:     "exampleuser",
 		Password: "examplepass",
-		DBname:   "exampledb",
+		DBname:   "server_db",
 	})
-
 	if err != nil {
 		return nil, err
 	}
 
-	s, err := LoadOrInit(db, startingID)
-	if err != nil {
-		return nil, err
-	}
-	return s, nil
-
-}
-
-func LoadOrInit(db *gorm.DB, startingID int) (*StorageContainer, error) {
-	// Check and set up the database
 	s := StorageContainer{RequestID: startingID, DB: db}
-	if s.hasTables() {
+	if err := s.SetupDB(); err != nil {
-		s.loadState()
+		return nil, err
-	} else {
-		if err := s.SetupDB(); err != nil {
-			return nil, err
-		}
-		s.setFresh()
 	}
+
+	if err = s.loadState(); err != nil {
+		return nil, err
+	}
+
 	return &s, nil
 }
 
@@ -80,12 +105,7 @@ func ConnectDB(s DBSpec) (*gorm.DB, error) {
 	return db, nil
 }
 
-func (s *StorageContainer) setFresh() {
-	// TODO Set initial state
-}
-
 func (s *StorageContainer) SetupDB() error {
-	// TODO Migrate the schemas
 	msg := "Failed to initialize database "
 
 	if err := s.DB.AutoMigrate(&DBInfo{}); err != nil {
@@ -108,9 +128,9 @@ func (s *StorageContainer) SetupDB() error {
 	return nil
 }
 
-func (s *StorageContainer) loadState() {
+func (s *StorageContainer) loadState() error {
 	// TODO load the state
-	return
+	return nil
 }
 
 func (s *StorageContainer) hasTables() bool {

pkg/storage/storage.go

@@ -28,6 +28,10 @@ func NewStorageSingle(startingID int) *StorageSingle {
 	return &s
 }
 
+func (s *StorageSingle) Setup(v *ServerStorageVisibles) {
+	s.modules = v
+}
+
 func (s *StorageSingle) NextID() int {
 	s.currentID += 1
 	return s.currentID

pkg/storage/types.go

@@ -13,6 +13,7 @@ type DBLocation struct {
 
 type StorageSingle struct {
 	currentID int
+	modules   *ServerStorageVisibles
 }
 
 type DBSpec struct {
@@ -59,4 +60,7 @@ type StorageContainer struct {
 	// Database version of StorageSingle
 	RequestID int
 	DB        *gorm.DB
+	modules   *ServerStorageVisibles
 }
+
+type ServerStorageVisibles struct{}