From 2d88b351ff28f6958901167bd67d1bd99976728c Mon Sep 17 00:00:00 2001 From: Michael Hohn Date: Tue, 11 Jun 2024 14:16:41 -0700 Subject: [PATCH] Introduce structs/interfacess for new storage units This commit simply splits the interfaces but introduces no new structs - Introduce the QueryPackStore, mrvacommander/pkg/qpstore - Introduce the CodeQL database store, pkg/qldbstore/interfaces.go --- README.md | 12 ++++ cmd/server/main.go | 5 +- pkg/agent/agent.go | 3 +- pkg/qldbstore/interfaces.go | 6 ++ pkg/qpstore/container.go | 138 ++++++++++++++++++++++++++++++++++++ pkg/qpstore/interfaces.go | 13 ++++ pkg/server/types.go | 3 +- pkg/storage/container.go | 27 ------- 8 files changed, 176 insertions(+), 31 deletions(-) create mode 100644 pkg/qldbstore/interfaces.go create mode 100644 pkg/qpstore/container.go create mode 100644 pkg/qpstore/interfaces.go diff --git a/README.md b/README.md index c476640..2d1f3ce 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,18 @@ These are simple steps using a single container. ### Manually create needed postgres databases +This is still necessary after `docker-compose up` to avoid + + [error] failed to initialize database, got error failed to connect to + `user=exampleuser database=server_db`: 172.25.0.3:5432 (postgres): server + error: FATAL: database "server_db" does not exist (SQLSTATE 3D000) + +from + + ./server -loglevel=debug -mode=container + +The steps: + # on the host psql -h localhost -p 5432 -U exampleuser -d postgres diff --git a/cmd/server/main.go b/cmd/server/main.go index 0a89951..3eb2d0d 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -13,6 +13,7 @@ import ( "mrvacommander/pkg/agent" "mrvacommander/pkg/logger" + "mrvacommander/pkg/qpstore" "mrvacommander/pkg/queue" "mrvacommander/pkg/server" "mrvacommander/pkg/storage" @@ -74,7 +75,7 @@ func main() { ss := storage.NewStorageSingle(config.Storage.StartingID) sr := agent.NewRunnerSingle(2, sq) // FIXME take value from configuration - qp, err := storage.NewQueryPackStore(config.Storage.StartingID) + qp, err := qpstore.NewStore(config.Storage.StartingID) if err != nil { slog.Error("Unable to initialize query pack storage") os.Exit(1) @@ -121,7 +122,7 @@ func main() { os.Exit(1) } - qp, err := storage.NewQueryPackStore(config.Storage.StartingID) + qp, err := qpstore.NewStore(config.Storage.StartingID) if err != nil { slog.Error("Unable to initialize query pack storage") os.Exit(1) diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 1005911..0592748 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -3,6 +3,7 @@ package agent import ( "mrvacommander/pkg/common" "mrvacommander/pkg/logger" + "mrvacommander/pkg/qpstore" "mrvacommander/pkg/queue" "mrvacommander/pkg/storage" @@ -36,7 +37,7 @@ type RunnerVisibles struct { Logger logger.Logger Queue queue.Queue // TODO extra package for query pack storage - QueryPackStore storage.Storage + QueryPackStore qpstore.Storage // TODO extra package for ql db storage QLDBStore storage.Storage } diff --git a/pkg/qldbstore/interfaces.go b/pkg/qldbstore/interfaces.go new file mode 100644 index 0000000..2cde68b --- /dev/null +++ b/pkg/qldbstore/interfaces.go @@ -0,0 +1,6 @@ +package qldbstore + +type DBLocation struct { + Prefix string + File string +} diff --git a/pkg/qpstore/container.go b/pkg/qpstore/container.go new file mode 100644 index 0000000..120b805 --- /dev/null +++ b/pkg/qpstore/container.go @@ -0,0 +1,138 @@ +package qpstore + +import ( + "fmt" + "log/slog" + "mrvacommander/pkg/common" + "mrvacommander/pkg/qldbstore" + "sync" + + "gorm.io/driver/postgres" + "gorm.io/gorm" +) + +var ( + DBmutex sync.Mutex +) + +type StorageVisibles struct{} + +type StorageContainer struct { + // Database version of StorageSingle + RequestID int + DB *gorm.DB + modules *StorageVisibles +} + +type DBSpec struct { + Host string + Port int + User string + Password string + DBname string +} + +func (s *StorageContainer) SetupDB() error { + // TODO set up query pack storage + return nil +} + +func (s *StorageContainer) LoadState() error { + // TODO load the state + return nil +} + +func (s *StorageContainer) hasTables() bool { + // TODO query to check for tables + return false +} + +func (s *StorageContainer) NextID() int { + // TODO update via db + return 12345 +} + +func (s *StorageContainer) SaveQueryPack(tgz []byte, sessionID int) (storagePath string, error error) { + // TODO save and return path + return "todo:no-path-yet", nil +} + +func (s *StorageContainer) FindAvailableDBs(analysisReposRequested []common.OwnerRepo) (notFoundRepos []common.OwnerRepo, analysisRepos *map[common.OwnerRepo]qldbstore.DBLocation) { + // TODO s.FindAvailableDBs() via postgres + analysisRepos = &map[common.OwnerRepo]qldbstore.DBLocation{} + notFoundRepos = []common.OwnerRepo{} + + return notFoundRepos, analysisRepos +} + +func (s *StorageContainer) Setup(v *StorageVisibles) { + s.modules = v +} + +func NewStore(startingID int) (Storage, error) { + // TODO drop the startingID + + db, err := ConnectDB(DBSpec{ + Host: "postgres", + Port: 5432, + User: "exampleuser", + Password: "examplepass", + DBname: "querypack_db", + }) + if err != nil { + return nil, err + } + + s := StorageContainer{RequestID: startingID, DB: db} + if err := s.SetupDB(); err != nil { + return nil, err + } + + if err = s.loadState(); err != nil { + return nil, err + } + + return &s, nil +} + +func NewStorageContainer(startingID int) (*StorageContainer, error) { + + db, err := ConnectDB(DBSpec{ + Host: "postgres", + Port: 5432, + User: "exampleuser", + Password: "examplepass", + DBname: "server_db", + }) + if err != nil { + return nil, err + } + + s := StorageContainer{RequestID: startingID, DB: db} + if err := s.SetupDB(); err != nil { + return nil, err + } + + if err = s.loadState(); err != nil { + return nil, err + } + + return &s, nil +} + +func ConnectDB(s DBSpec) (*gorm.DB, error) { + // Open the database connection + dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", + s.Host, s.Port, s.User, s.Password, s.DBname) + db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{}) + if err != nil { + slog.Error("Error connecting to the database", "err", err) + return nil, err + } + return db, nil +} + +func (s *StorageContainer) loadState() error { + // TODO load the state + return nil +} diff --git a/pkg/qpstore/interfaces.go b/pkg/qpstore/interfaces.go new file mode 100644 index 0000000..26081fc --- /dev/null +++ b/pkg/qpstore/interfaces.go @@ -0,0 +1,13 @@ +package qpstore + +import ( + "mrvacommander/pkg/common" + "mrvacommander/pkg/qldbstore" +) + +type Storage interface { + NextID() int + SaveQueryPack(tgz []byte, sessionID int) (storagePath string, error error) + FindAvailableDBs(analysisReposRequested []common.OwnerRepo) (not_found_repos []common.OwnerRepo, + analysisRepos *map[common.OwnerRepo]qldbstore.DBLocation) +} diff --git a/pkg/server/types.go b/pkg/server/types.go index 0f221e9..7723c89 100644 --- a/pkg/server/types.go +++ b/pkg/server/types.go @@ -3,6 +3,7 @@ package server import ( "mrvacommander/pkg/common" "mrvacommander/pkg/logger" + "mrvacommander/pkg/qpstore" "mrvacommander/pkg/queue" "mrvacommander/pkg/storage" ) @@ -46,7 +47,7 @@ type CommanderVisibles struct { Queue queue.Queue ServerStore storage.Storage // TODO extra package for query pack storage - QueryPackStore storage.Storage + QueryPackStore qpstore.Storage // TODO extra package for ql db storage QLDBStore storage.Storage } diff --git a/pkg/storage/container.go b/pkg/storage/container.go index 65eb4ea..8a3febb 100644 --- a/pkg/storage/container.go +++ b/pkg/storage/container.go @@ -41,33 +41,6 @@ func NewQLDBStore() (*StorageContainer, error) { return nil, nil } -func NewQueryPackStore(startingID int) (*StorageContainer, error) { - // TODO set up querypack_db - // TODO drop the startingID - - db, err := ConnectDB(DBSpec{ - Host: "postgres", - Port: 5432, - User: "exampleuser", - Password: "examplepass", - DBname: "querypack_db", - }) - if err != nil { - return nil, err - } - - s := StorageContainer{RequestID: startingID, DB: db} - if err := s.SetupDB(); err != nil { - return nil, err - } - - if err = s.loadState(); err != nil { - return nil, err - } - - return &s, nil -} - func NewStorageContainer(startingID int) (*StorageContainer, error) { db, err := ConnectDB(DBSpec{