From 7e0d6909da53ff6feb4951399964460caa107b09 Mon Sep 17 00:00:00 2001 From: Michael Hohn Date: Fri, 7 Jun 2024 13:14:41 -0700 Subject: [PATCH] wip: Make cross-module visibility explicit via Visibles structs All access is/will be through interfaces accessed through these structs. This introduces several distinct storage units: + DB for server state + DB for codeql databases + query pack store The steps for manually creating needed databases are in the README --- README.md | 27 +++++++++++++ cmd/server/main.go | 83 ++++++++++++++++++++++++++++++++-------- pkg/agent/agent.go | 14 +++++++ pkg/logger/types.go | 7 ++++ pkg/queue/types.go | 14 ++++++- pkg/server/server.go | 8 ++-- pkg/server/types.go | 28 +++++++++----- pkg/storage/container.go | 72 +++++++++++++++++++++------------- pkg/storage/storage.go | 4 ++ pkg/storage/types.go | 4 ++ 10 files changed, 205 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 903d901..e368301 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,33 @@ Some postgres specific commands \dt +1. Examine a table + + select * from db_infos + +1. Show all columns in a specific table + + \d+ db_infos + +1. Miscellany + + \pset pager off + \lo_import FILE [COMMENT] + + +Manually create needed postgres databases + + # on the host + psql -h localhost -p 5432 -U exampleuser -d postgres + + # Conditionally create dbs + SELECT 'CREATE DATABASE server_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'server_db')\gexec + SELECT 'CREATE DATABASE querypack_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'querypack_db')\gexec + SELECT 'CREATE DATABASE qldb_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'qldb_db')\gexec + + # List all dbs + \l + To run pgmin, the minimal go/postgres test part of this repository: 1. Run pgmin diff --git a/cmd/server/main.go b/cmd/server/main.go index b3a9720..adc0d44 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -74,38 +74,89 @@ func main() { ss := storage.NewStorageSingle(config.Storage.StartingID) sr := agent.NewRunnerSingle(2, sq) // FIXME take value from configuration - state := server.State{ - Commander: sc, - Logger: sl, - Queue: sq, - Storage: ss, - Runner: sr, + qp, err := storage.NewQueryPackStore(config.Storage.StartingID) + if err != nil { + slog.Error("Unable to initialize query pack storage") + os.Exit(1) } - sc.Setup(&state) // sc is part of state and dereferences it + ql, err := storage.NewQLDBStore() + if err != nil { + slog.Error("Unable to initialize ql database storage") + os.Exit(1) + } + + sc.Setup(&server.CommanderVisibles{ + Logger: sl, + Queue: sq, + ServerStore: ss, + QueryPackStore: qp, + QLDBStore: ql, + }) + + sl.Setup(&logger.LoggerVisibles{}) + + sq.Setup(&queue.QueueVisibles{ + Logger: sl, + }) + + ss.Setup(&storage.ServerStorageVisibles{}) + + sr.Setup(&agent.RunnerVisibles{ + Logger: sl, + Queue: sq, + QueryPackStore: qp, + QLDBStore: ql, + }) case "container": // Assemble container version sq := queue.NewQueueSingle(2) // FIXME take value from configuration sc := server.NewCommanderSingle(nil, sq) sl := logger.NewLoggerSingle() + ss, err := storage.NewStorageContainer(config.Storage.StartingID) if err != nil { - slog.Error("Unable to initialize storage") + slog.Error("Unable to initialize server storage") + os.Exit(1) + } + + qp, err := storage.NewQueryPackStore(config.Storage.StartingID) + if err != nil { + slog.Error("Unable to initialize query pack storage") + os.Exit(1) + } + + ql, err := storage.NewQLDBStore() + if err != nil { + slog.Error("Unable to initialize ql database storage") os.Exit(1) } sr := agent.NewRunnerSingle(2, sq) // FIXME take value from configuration - state := server.State{ - Commander: sc, - Logger: sl, - Queue: sq, - Storage: ss, - Runner: sr, - } + sc.Setup(&server.CommanderVisibles{ + Logger: sl, + Queue: sq, + ServerStore: ss, + QueryPackStore: qp, + QLDBStore: ql, + }) - sc.Setup(&state) // sc is part of state and dereferences it + sl.Setup(&logger.LoggerVisibles{}) + + sq.Setup(&queue.QueueVisibles{ + Logger: sl, + }) + + ss.Setup(&storage.ServerStorageVisibles{}) + + sr.Setup(&agent.RunnerVisibles{ + Logger: sl, + Queue: sq, + QueryPackStore: qp, + QLDBStore: ql, + }) case "cluster": // Assemble cccluster diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 0ccee85..1005911 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -2,6 +2,7 @@ package agent import ( "mrvacommander/pkg/common" + "mrvacommander/pkg/logger" "mrvacommander/pkg/queue" "mrvacommander/pkg/storage" @@ -31,6 +32,19 @@ func NewRunnerSingle(numWorkers int, queue queue.Queue) *RunnerSingle { return &r } +type RunnerVisibles struct { + Logger logger.Logger + Queue queue.Queue + // TODO extra package for query pack storage + QueryPackStore storage.Storage + // TODO extra package for ql db storage + QLDBStore storage.Storage +} + +func (c *RunnerSingle) Setup(st *RunnerVisibles) { + return +} + func (r *RunnerSingle) worker(wid int) { var job common.AnalyzeJob diff --git a/pkg/logger/types.go b/pkg/logger/types.go index 51afee8..7fe3bbd 100644 --- a/pkg/logger/types.go +++ b/pkg/logger/types.go @@ -1,9 +1,16 @@ package logger type LoggerSingle struct { + modules *LoggerVisibles } func NewLoggerSingle() *LoggerSingle { l := LoggerSingle{} return &l } + +type LoggerVisibles struct{} + +func (l *LoggerSingle) Setup(v *LoggerVisibles) { + l.modules = v +} diff --git a/pkg/queue/types.go b/pkg/queue/types.go index ea39e74..b8f5406 100644 --- a/pkg/queue/types.go +++ b/pkg/queue/types.go @@ -1,11 +1,23 @@ package queue -import "mrvacommander/pkg/common" +import ( + "mrvacommander/pkg/common" + "mrvacommander/pkg/logger" +) type QueueSingle struct { NumWorkers int jobs chan common.AnalyzeJob results chan common.AnalyzeResult + modules *QueueVisibles +} + +type QueueVisibles struct { + Logger logger.Logger +} + +func (q *QueueSingle) Setup(v *QueueVisibles) { + q.modules = v } func NewQueueSingle(numWorkers int) *QueueSingle { diff --git a/pkg/server/server.go b/pkg/server/server.go index 1bc72ae..f8365a9 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -21,7 +21,7 @@ import ( "github.com/gorilla/mux" ) -func (c *CommanderSingle) Setup(st *State) { +func (c *CommanderSingle) Setup(st *CommanderVisibles) { r := mux.NewRouter() c.st = st @@ -275,7 +275,7 @@ func (c *CommanderSingle) MirvaRequest(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) slog.Info("New mrva run ", "owner", vars["owner"], "repo", vars["repo"]) - session_id := c.st.Storage.NextID() + session_id := c.st.ServerStore.NextID() session_owner := vars["owner"] session_controller_repo := vars["repo"] slog.Info("new run", "id: ", fmt.Sprint(session_id), session_owner, session_controller_repo) @@ -284,7 +284,7 @@ func (c *CommanderSingle) MirvaRequest(w http.ResponseWriter, r *http.Request) { return } - not_found_repos, analysisRepos := c.st.Storage.FindAvailableDBs(session_repositories) + not_found_repos, analysisRepos := c.st.ServerStore.FindAvailableDBs(session_repositories) c.queue.StartAnalyses(analysisRepos, session_id, session_language) @@ -492,7 +492,7 @@ func (c *CommanderSingle) extract_tgz(qp string, sessionID int) (string, error) return "", err } - session_query_pack_tgz_filepath, err := c.st.Storage.SaveQueryPack(tgz, sessionID) + session_query_pack_tgz_filepath, err := c.st.ServerStore.SaveQueryPack(tgz, sessionID) if err != nil { return "", err } diff --git a/pkg/server/types.go b/pkg/server/types.go index 6d875c2..0fea515 100644 --- a/pkg/server/types.go +++ b/pkg/server/types.go @@ -1,7 +1,6 @@ package server import ( - "mrvacommander/pkg/agent" "mrvacommander/pkg/common" "mrvacommander/pkg/logger" "mrvacommander/pkg/queue" @@ -26,19 +25,30 @@ type SessionInfo struct { } type CommanderSingle struct { - st *State + st *CommanderVisibles + // TODO remove: queue queue.Queue } -func NewCommanderSingle(s *State, q queue.Queue) *CommanderSingle { +func NewCommanderSingle(s *CommanderVisibles, q queue.Queue) *CommanderSingle { c := CommanderSingle{s, q} return &c } -type State struct { - Commander Commander - Logger logger.Logger - Queue queue.Queue - Storage storage.Storage - Runner agent.Runner +// type State struct { +// Commander Commander +// Logger logger.Logger +// Queue queue.Queue +// Storage storage.Storage +// Runner agent.Runner +// } + +type CommanderVisibles struct { + Logger logger.Logger + Queue queue.Queue + ServerStore storage.Storage + // TODO extra package for query pack storage + QueryPackStore storage.Storage + // TODO extra package for ql db storage + QLDBStore storage.Storage } diff --git a/pkg/storage/container.go b/pkg/storage/container.go index fe83190..65eb4ea 100644 --- a/pkg/storage/container.go +++ b/pkg/storage/container.go @@ -32,6 +32,42 @@ func (s *StorageContainer) FindAvailableDBs(analysisReposRequested []common.Owne return notFoundRepos, analysisRepos } +func (s *StorageContainer) Setup(v *ServerStorageVisibles) { + s.modules = v +} + +func NewQLDBStore() (*StorageContainer, error) { + // TODO set up qldb_db + return nil, nil +} + +func NewQueryPackStore(startingID int) (*StorageContainer, error) { + // TODO set up querypack_db + // TODO drop the startingID + + db, err := ConnectDB(DBSpec{ + Host: "postgres", + Port: 5432, + User: "exampleuser", + Password: "examplepass", + DBname: "querypack_db", + }) + if err != nil { + return nil, err + } + + s := StorageContainer{RequestID: startingID, DB: db} + if err := s.SetupDB(); err != nil { + return nil, err + } + + if err = s.loadState(); err != nil { + return nil, err + } + + return &s, nil +} + func NewStorageContainer(startingID int) (*StorageContainer, error) { db, err := ConnectDB(DBSpec{ @@ -39,32 +75,21 @@ func NewStorageContainer(startingID int) (*StorageContainer, error) { Port: 5432, User: "exampleuser", Password: "examplepass", - DBname: "exampledb", + DBname: "server_db", }) - if err != nil { return nil, err } - s, err := LoadOrInit(db, startingID) - if err != nil { - return nil, err - } - return s, nil - -} - -func LoadOrInit(db *gorm.DB, startingID int) (*StorageContainer, error) { - // Check and set up the database s := StorageContainer{RequestID: startingID, DB: db} - if s.hasTables() { - s.loadState() - } else { - if err := s.SetupDB(); err != nil { - return nil, err - } - s.setFresh() + if err := s.SetupDB(); err != nil { + return nil, err } + + if err = s.loadState(); err != nil { + return nil, err + } + return &s, nil } @@ -80,12 +105,7 @@ func ConnectDB(s DBSpec) (*gorm.DB, error) { return db, nil } -func (s *StorageContainer) setFresh() { - // TODO Set initial state -} - func (s *StorageContainer) SetupDB() error { - // TODO Migrate the schemas msg := "Failed to initialize database " if err := s.DB.AutoMigrate(&DBInfo{}); err != nil { @@ -108,9 +128,9 @@ func (s *StorageContainer) SetupDB() error { return nil } -func (s *StorageContainer) loadState() { +func (s *StorageContainer) loadState() error { // TODO load the state - return + return nil } func (s *StorageContainer) hasTables() bool { diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index df3707b..1fe6dda 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -28,6 +28,10 @@ func NewStorageSingle(startingID int) *StorageSingle { return &s } +func (s *StorageSingle) Setup(v *ServerStorageVisibles) { + s.modules = v +} + func (s *StorageSingle) NextID() int { s.currentID += 1 return s.currentID diff --git a/pkg/storage/types.go b/pkg/storage/types.go index 77346fc..7c94c02 100644 --- a/pkg/storage/types.go +++ b/pkg/storage/types.go @@ -13,6 +13,7 @@ type DBLocation struct { type StorageSingle struct { currentID int + modules *ServerStorageVisibles } type DBSpec struct { @@ -59,4 +60,7 @@ type StorageContainer struct { // Database version of StorageSingle RequestID int DB *gorm.DB + modules *ServerStorageVisibles } + +type ServerStorageVisibles struct{}