From 7e0d6909da53ff6feb4951399964460caa107b09 Mon Sep 17 00:00:00 2001 From: Michael Hohn Date: Fri, 7 Jun 2024 13:14:41 -0700 Subject: [PATCH 1/7] wip: Make cross-module visibility explicit via Visibles structs All access is/will be through interfaces accessed through these structs. This introduces several distinct storage units: + DB for server state + DB for codeql databases + query pack store The steps for manually creating needed databases are in the README --- README.md | 27 +++++++++++++ cmd/server/main.go | 83 ++++++++++++++++++++++++++++++++-------- pkg/agent/agent.go | 14 +++++++ pkg/logger/types.go | 7 ++++ pkg/queue/types.go | 14 ++++++- pkg/server/server.go | 8 ++-- pkg/server/types.go | 28 +++++++++----- pkg/storage/container.go | 72 +++++++++++++++++++++------------- pkg/storage/storage.go | 4 ++ pkg/storage/types.go | 4 ++ 10 files changed, 205 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 903d901..e368301 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,33 @@ Some postgres specific commands \dt +1. Examine a table + + select * from db_infos + +1. Show all columns in a specific table + + \d+ db_infos + +1. Miscellany + + \pset pager off + \lo_import FILE [COMMENT] + + +Manually create needed postgres databases + + # on the host + psql -h localhost -p 5432 -U exampleuser -d postgres + + # Conditionally create dbs + SELECT 'CREATE DATABASE server_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'server_db')\gexec + SELECT 'CREATE DATABASE querypack_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'querypack_db')\gexec + SELECT 'CREATE DATABASE qldb_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'qldb_db')\gexec + + # List all dbs + \l + To run pgmin, the minimal go/postgres test part of this repository: 1. Run pgmin diff --git a/cmd/server/main.go b/cmd/server/main.go index b3a9720..adc0d44 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -74,38 +74,89 @@ func main() { ss := storage.NewStorageSingle(config.Storage.StartingID) sr := agent.NewRunnerSingle(2, sq) // FIXME take value from configuration - state := server.State{ - Commander: sc, - Logger: sl, - Queue: sq, - Storage: ss, - Runner: sr, + qp, err := storage.NewQueryPackStore(config.Storage.StartingID) + if err != nil { + slog.Error("Unable to initialize query pack storage") + os.Exit(1) } - sc.Setup(&state) // sc is part of state and dereferences it + ql, err := storage.NewQLDBStore() + if err != nil { + slog.Error("Unable to initialize ql database storage") + os.Exit(1) + } + + sc.Setup(&server.CommanderVisibles{ + Logger: sl, + Queue: sq, + ServerStore: ss, + QueryPackStore: qp, + QLDBStore: ql, + }) + + sl.Setup(&logger.LoggerVisibles{}) + + sq.Setup(&queue.QueueVisibles{ + Logger: sl, + }) + + ss.Setup(&storage.ServerStorageVisibles{}) + + sr.Setup(&agent.RunnerVisibles{ + Logger: sl, + Queue: sq, + QueryPackStore: qp, + QLDBStore: ql, + }) case "container": // Assemble container version sq := queue.NewQueueSingle(2) // FIXME take value from configuration sc := server.NewCommanderSingle(nil, sq) sl := logger.NewLoggerSingle() + ss, err := storage.NewStorageContainer(config.Storage.StartingID) if err != nil { - slog.Error("Unable to initialize storage") + slog.Error("Unable to initialize server storage") + os.Exit(1) + } + + qp, err := storage.NewQueryPackStore(config.Storage.StartingID) + if err != nil { + slog.Error("Unable to initialize query pack storage") + os.Exit(1) + } + + ql, err := storage.NewQLDBStore() + if err != nil { + slog.Error("Unable to initialize ql database storage") os.Exit(1) } sr := agent.NewRunnerSingle(2, sq) // FIXME take value from configuration - state := server.State{ - Commander: sc, - Logger: sl, - Queue: sq, - Storage: ss, - Runner: sr, - } + sc.Setup(&server.CommanderVisibles{ + Logger: sl, + Queue: sq, + ServerStore: ss, + QueryPackStore: qp, + QLDBStore: ql, + }) - sc.Setup(&state) // sc is part of state and dereferences it + sl.Setup(&logger.LoggerVisibles{}) + + sq.Setup(&queue.QueueVisibles{ + Logger: sl, + }) + + ss.Setup(&storage.ServerStorageVisibles{}) + + sr.Setup(&agent.RunnerVisibles{ + Logger: sl, + Queue: sq, + QueryPackStore: qp, + QLDBStore: ql, + }) case "cluster": // Assemble cccluster diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 0ccee85..1005911 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -2,6 +2,7 @@ package agent import ( "mrvacommander/pkg/common" + "mrvacommander/pkg/logger" "mrvacommander/pkg/queue" "mrvacommander/pkg/storage" @@ -31,6 +32,19 @@ func NewRunnerSingle(numWorkers int, queue queue.Queue) *RunnerSingle { return &r } +type RunnerVisibles struct { + Logger logger.Logger + Queue queue.Queue + // TODO extra package for query pack storage + QueryPackStore storage.Storage + // TODO extra package for ql db storage + QLDBStore storage.Storage +} + +func (c *RunnerSingle) Setup(st *RunnerVisibles) { + return +} + func (r *RunnerSingle) worker(wid int) { var job common.AnalyzeJob diff --git a/pkg/logger/types.go b/pkg/logger/types.go index 51afee8..7fe3bbd 100644 --- a/pkg/logger/types.go +++ b/pkg/logger/types.go @@ -1,9 +1,16 @@ package logger type LoggerSingle struct { + modules *LoggerVisibles } func NewLoggerSingle() *LoggerSingle { l := LoggerSingle{} return &l } + +type LoggerVisibles struct{} + +func (l *LoggerSingle) Setup(v *LoggerVisibles) { + l.modules = v +} diff --git a/pkg/queue/types.go b/pkg/queue/types.go index ea39e74..b8f5406 100644 --- a/pkg/queue/types.go +++ b/pkg/queue/types.go @@ -1,11 +1,23 @@ package queue -import "mrvacommander/pkg/common" +import ( + "mrvacommander/pkg/common" + "mrvacommander/pkg/logger" +) type QueueSingle struct { NumWorkers int jobs chan common.AnalyzeJob results chan common.AnalyzeResult + modules *QueueVisibles +} + +type QueueVisibles struct { + Logger logger.Logger +} + +func (q *QueueSingle) Setup(v *QueueVisibles) { + q.modules = v } func NewQueueSingle(numWorkers int) *QueueSingle { diff --git a/pkg/server/server.go b/pkg/server/server.go index 1bc72ae..f8365a9 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -21,7 +21,7 @@ import ( "github.com/gorilla/mux" ) -func (c *CommanderSingle) Setup(st *State) { +func (c *CommanderSingle) Setup(st *CommanderVisibles) { r := mux.NewRouter() c.st = st @@ -275,7 +275,7 @@ func (c *CommanderSingle) MirvaRequest(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) slog.Info("New mrva run ", "owner", vars["owner"], "repo", vars["repo"]) - session_id := c.st.Storage.NextID() + session_id := c.st.ServerStore.NextID() session_owner := vars["owner"] session_controller_repo := vars["repo"] slog.Info("new run", "id: ", fmt.Sprint(session_id), session_owner, session_controller_repo) @@ -284,7 +284,7 @@ func (c *CommanderSingle) MirvaRequest(w http.ResponseWriter, r *http.Request) { return } - not_found_repos, analysisRepos := c.st.Storage.FindAvailableDBs(session_repositories) + not_found_repos, analysisRepos := c.st.ServerStore.FindAvailableDBs(session_repositories) c.queue.StartAnalyses(analysisRepos, session_id, session_language) @@ -492,7 +492,7 @@ func (c *CommanderSingle) extract_tgz(qp string, sessionID int) (string, error) return "", err } - session_query_pack_tgz_filepath, err := c.st.Storage.SaveQueryPack(tgz, sessionID) + session_query_pack_tgz_filepath, err := c.st.ServerStore.SaveQueryPack(tgz, sessionID) if err != nil { return "", err } diff --git a/pkg/server/types.go b/pkg/server/types.go index 6d875c2..0fea515 100644 --- a/pkg/server/types.go +++ b/pkg/server/types.go @@ -1,7 +1,6 @@ package server import ( - "mrvacommander/pkg/agent" "mrvacommander/pkg/common" "mrvacommander/pkg/logger" "mrvacommander/pkg/queue" @@ -26,19 +25,30 @@ type SessionInfo struct { } type CommanderSingle struct { - st *State + st *CommanderVisibles + // TODO remove: queue queue.Queue } -func NewCommanderSingle(s *State, q queue.Queue) *CommanderSingle { +func NewCommanderSingle(s *CommanderVisibles, q queue.Queue) *CommanderSingle { c := CommanderSingle{s, q} return &c } -type State struct { - Commander Commander - Logger logger.Logger - Queue queue.Queue - Storage storage.Storage - Runner agent.Runner +// type State struct { +// Commander Commander +// Logger logger.Logger +// Queue queue.Queue +// Storage storage.Storage +// Runner agent.Runner +// } + +type CommanderVisibles struct { + Logger logger.Logger + Queue queue.Queue + ServerStore storage.Storage + // TODO extra package for query pack storage + QueryPackStore storage.Storage + // TODO extra package for ql db storage + QLDBStore storage.Storage } diff --git a/pkg/storage/container.go b/pkg/storage/container.go index fe83190..65eb4ea 100644 --- a/pkg/storage/container.go +++ b/pkg/storage/container.go @@ -32,6 +32,42 @@ func (s *StorageContainer) FindAvailableDBs(analysisReposRequested []common.Owne return notFoundRepos, analysisRepos } +func (s *StorageContainer) Setup(v *ServerStorageVisibles) { + s.modules = v +} + +func NewQLDBStore() (*StorageContainer, error) { + // TODO set up qldb_db + return nil, nil +} + +func NewQueryPackStore(startingID int) (*StorageContainer, error) { + // TODO set up querypack_db + // TODO drop the startingID + + db, err := ConnectDB(DBSpec{ + Host: "postgres", + Port: 5432, + User: "exampleuser", + Password: "examplepass", + DBname: "querypack_db", + }) + if err != nil { + return nil, err + } + + s := StorageContainer{RequestID: startingID, DB: db} + if err := s.SetupDB(); err != nil { + return nil, err + } + + if err = s.loadState(); err != nil { + return nil, err + } + + return &s, nil +} + func NewStorageContainer(startingID int) (*StorageContainer, error) { db, err := ConnectDB(DBSpec{ @@ -39,32 +75,21 @@ func NewStorageContainer(startingID int) (*StorageContainer, error) { Port: 5432, User: "exampleuser", Password: "examplepass", - DBname: "exampledb", + DBname: "server_db", }) - if err != nil { return nil, err } - s, err := LoadOrInit(db, startingID) - if err != nil { - return nil, err - } - return s, nil - -} - -func LoadOrInit(db *gorm.DB, startingID int) (*StorageContainer, error) { - // Check and set up the database s := StorageContainer{RequestID: startingID, DB: db} - if s.hasTables() { - s.loadState() - } else { - if err := s.SetupDB(); err != nil { - return nil, err - } - s.setFresh() + if err := s.SetupDB(); err != nil { + return nil, err } + + if err = s.loadState(); err != nil { + return nil, err + } + return &s, nil } @@ -80,12 +105,7 @@ func ConnectDB(s DBSpec) (*gorm.DB, error) { return db, nil } -func (s *StorageContainer) setFresh() { - // TODO Set initial state -} - func (s *StorageContainer) SetupDB() error { - // TODO Migrate the schemas msg := "Failed to initialize database " if err := s.DB.AutoMigrate(&DBInfo{}); err != nil { @@ -108,9 +128,9 @@ func (s *StorageContainer) SetupDB() error { return nil } -func (s *StorageContainer) loadState() { +func (s *StorageContainer) loadState() error { // TODO load the state - return + return nil } func (s *StorageContainer) hasTables() bool { diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index df3707b..1fe6dda 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -28,6 +28,10 @@ func NewStorageSingle(startingID int) *StorageSingle { return &s } +func (s *StorageSingle) Setup(v *ServerStorageVisibles) { + s.modules = v +} + func (s *StorageSingle) NextID() int { s.currentID += 1 return s.currentID diff --git a/pkg/storage/types.go b/pkg/storage/types.go index 77346fc..7c94c02 100644 --- a/pkg/storage/types.go +++ b/pkg/storage/types.go @@ -13,6 +13,7 @@ type DBLocation struct { type StorageSingle struct { currentID int + modules *ServerStorageVisibles } type DBSpec struct { @@ -59,4 +60,7 @@ type StorageContainer struct { // Database version of StorageSingle RequestID int DB *gorm.DB + modules *ServerStorageVisibles } + +type ServerStorageVisibles struct{} From fc29fc5653afe9fbe13aa15bc0d8afa76ac8cbe4 Mon Sep 17 00:00:00 2001 From: Michael Hohn Date: Tue, 11 Jun 2024 13:19:05 -0700 Subject: [PATCH 2/7] wip: update passing Queue to Commander - Add minio to docker-compose - Fix use of server.NewCommanderSingle --- README.md | 11 ++++------- cmd/server/main.go | 4 ++-- docker-compose.yml | 24 +++++++++++++++++++++--- pkg/server/server.go | 2 +- pkg/server/types.go | 6 ++---- 5 files changed, 30 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index e368301..c476640 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ These are simple steps using a single container. cd /mrva/mrvacommander/cmd/server/ && ./server ## Using docker-compose -Steps to build and run the server in a multi-container environment set up by docker-compose. +### Steps to build and run the server in a multi-container environment set up by docker-compose. 1. Build server on host @@ -51,11 +51,8 @@ Steps to build and run the server in a multi-container environment set up by doc ./server -loglevel=debug -mode=container 1. Test server via remote client by following the steps in [gh-mrva](https://github.com/hohn/gh-mrva/blob/connection-redirect/README.org#compacted-edit-run-debug-cycle) - - - -Some general docker-compose commands +### Some general docker-compose commands 2. Get service status @@ -76,7 +73,7 @@ Some general docker-compose commands curl -I http://rabbitmq:15672 -Some postgres specific commands +### Some postgres specific commands 1. Access PostgreSQL @@ -100,7 +97,7 @@ Some postgres specific commands \lo_import FILE [COMMENT] -Manually create needed postgres databases +### Manually create needed postgres databases # on the host psql -h localhost -p 5432 -U exampleuser -d postgres diff --git a/cmd/server/main.go b/cmd/server/main.go index adc0d44..0a89951 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -69,7 +69,7 @@ func main() { case "standalone": // Assemble single-process version sq := queue.NewQueueSingle(2) // FIXME take value from configuration - sc := server.NewCommanderSingle(nil, sq) + sc := server.NewCommanderSingle() sl := logger.NewLoggerSingle() ss := storage.NewStorageSingle(config.Storage.StartingID) sr := agent.NewRunnerSingle(2, sq) // FIXME take value from configuration @@ -112,7 +112,7 @@ func main() { case "container": // Assemble container version sq := queue.NewQueueSingle(2) // FIXME take value from configuration - sc := server.NewCommanderSingle(nil, sq) + sc := server.NewCommanderSingle() sl := logger.NewLoggerSingle() ss, err := storage.NewStorageContainer(config.Storage.StartingID) diff --git a/docker-compose.yml b/docker-compose.yml index e21143c..3620c1a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,10 +47,28 @@ services: networks: - backend + minio: + image: minio/minio:RELEASE.2024-06-11T03-13-30Z + container_name: minio + ports: + - "9000:9000" + - "9001:9001" + environment: + MINIO_ROOT_USER: user + MINIO_ROOT_PASSWORD: musty-coke + command: server /data --console-address ":9001" + volumes: + - minio-data:/data + +volumes: + minio-data: + postgres_data: + driver: local + networks: backend: driver: bridge -volumes: - postgres_data: - driver: local + + + diff --git a/pkg/server/server.go b/pkg/server/server.go index f8365a9..312af2b 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -286,7 +286,7 @@ func (c *CommanderSingle) MirvaRequest(w http.ResponseWriter, r *http.Request) { not_found_repos, analysisRepos := c.st.ServerStore.FindAvailableDBs(session_repositories) - c.queue.StartAnalyses(analysisRepos, session_id, session_language) + c.st.Queue.StartAnalyses(analysisRepos, session_id, session_language) si := SessionInfo{ ID: session_id, diff --git a/pkg/server/types.go b/pkg/server/types.go index 0fea515..0f221e9 100644 --- a/pkg/server/types.go +++ b/pkg/server/types.go @@ -26,12 +26,10 @@ type SessionInfo struct { type CommanderSingle struct { st *CommanderVisibles - // TODO remove: - queue queue.Queue } -func NewCommanderSingle(s *CommanderVisibles, q queue.Queue) *CommanderSingle { - c := CommanderSingle{s, q} +func NewCommanderSingle() *CommanderSingle { + c := CommanderSingle{} return &c } From 2d88b351ff28f6958901167bd67d1bd99976728c Mon Sep 17 00:00:00 2001 From: Michael Hohn Date: Tue, 11 Jun 2024 14:16:41 -0700 Subject: [PATCH 3/7] Introduce structs/interfacess for new storage units This commit simply splits the interfaces but introduces no new structs - Introduce the QueryPackStore, mrvacommander/pkg/qpstore - Introduce the CodeQL database store, pkg/qldbstore/interfaces.go --- README.md | 12 ++++ cmd/server/main.go | 5 +- pkg/agent/agent.go | 3 +- pkg/qldbstore/interfaces.go | 6 ++ pkg/qpstore/container.go | 138 ++++++++++++++++++++++++++++++++++++ pkg/qpstore/interfaces.go | 13 ++++ pkg/server/types.go | 3 +- pkg/storage/container.go | 27 ------- 8 files changed, 176 insertions(+), 31 deletions(-) create mode 100644 pkg/qldbstore/interfaces.go create mode 100644 pkg/qpstore/container.go create mode 100644 pkg/qpstore/interfaces.go diff --git a/README.md b/README.md index c476640..2d1f3ce 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,18 @@ These are simple steps using a single container. ### Manually create needed postgres databases +This is still necessary after `docker-compose up` to avoid + + [error] failed to initialize database, got error failed to connect to + `user=exampleuser database=server_db`: 172.25.0.3:5432 (postgres): server + error: FATAL: database "server_db" does not exist (SQLSTATE 3D000) + +from + + ./server -loglevel=debug -mode=container + +The steps: + # on the host psql -h localhost -p 5432 -U exampleuser -d postgres diff --git a/cmd/server/main.go b/cmd/server/main.go index 0a89951..3eb2d0d 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -13,6 +13,7 @@ import ( "mrvacommander/pkg/agent" "mrvacommander/pkg/logger" + "mrvacommander/pkg/qpstore" "mrvacommander/pkg/queue" "mrvacommander/pkg/server" "mrvacommander/pkg/storage" @@ -74,7 +75,7 @@ func main() { ss := storage.NewStorageSingle(config.Storage.StartingID) sr := agent.NewRunnerSingle(2, sq) // FIXME take value from configuration - qp, err := storage.NewQueryPackStore(config.Storage.StartingID) + qp, err := qpstore.NewStore(config.Storage.StartingID) if err != nil { slog.Error("Unable to initialize query pack storage") os.Exit(1) @@ -121,7 +122,7 @@ func main() { os.Exit(1) } - qp, err := storage.NewQueryPackStore(config.Storage.StartingID) + qp, err := qpstore.NewStore(config.Storage.StartingID) if err != nil { slog.Error("Unable to initialize query pack storage") os.Exit(1) diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 1005911..0592748 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -3,6 +3,7 @@ package agent import ( "mrvacommander/pkg/common" "mrvacommander/pkg/logger" + "mrvacommander/pkg/qpstore" "mrvacommander/pkg/queue" "mrvacommander/pkg/storage" @@ -36,7 +37,7 @@ type RunnerVisibles struct { Logger logger.Logger Queue queue.Queue // TODO extra package for query pack storage - QueryPackStore storage.Storage + QueryPackStore qpstore.Storage // TODO extra package for ql db storage QLDBStore storage.Storage } diff --git a/pkg/qldbstore/interfaces.go b/pkg/qldbstore/interfaces.go new file mode 100644 index 0000000..2cde68b --- /dev/null +++ b/pkg/qldbstore/interfaces.go @@ -0,0 +1,6 @@ +package qldbstore + +type DBLocation struct { + Prefix string + File string +} diff --git a/pkg/qpstore/container.go b/pkg/qpstore/container.go new file mode 100644 index 0000000..120b805 --- /dev/null +++ b/pkg/qpstore/container.go @@ -0,0 +1,138 @@ +package qpstore + +import ( + "fmt" + "log/slog" + "mrvacommander/pkg/common" + "mrvacommander/pkg/qldbstore" + "sync" + + "gorm.io/driver/postgres" + "gorm.io/gorm" +) + +var ( + DBmutex sync.Mutex +) + +type StorageVisibles struct{} + +type StorageContainer struct { + // Database version of StorageSingle + RequestID int + DB *gorm.DB + modules *StorageVisibles +} + +type DBSpec struct { + Host string + Port int + User string + Password string + DBname string +} + +func (s *StorageContainer) SetupDB() error { + // TODO set up query pack storage + return nil +} + +func (s *StorageContainer) LoadState() error { + // TODO load the state + return nil +} + +func (s *StorageContainer) hasTables() bool { + // TODO query to check for tables + return false +} + +func (s *StorageContainer) NextID() int { + // TODO update via db + return 12345 +} + +func (s *StorageContainer) SaveQueryPack(tgz []byte, sessionID int) (storagePath string, error error) { + // TODO save and return path + return "todo:no-path-yet", nil +} + +func (s *StorageContainer) FindAvailableDBs(analysisReposRequested []common.OwnerRepo) (notFoundRepos []common.OwnerRepo, analysisRepos *map[common.OwnerRepo]qldbstore.DBLocation) { + // TODO s.FindAvailableDBs() via postgres + analysisRepos = &map[common.OwnerRepo]qldbstore.DBLocation{} + notFoundRepos = []common.OwnerRepo{} + + return notFoundRepos, analysisRepos +} + +func (s *StorageContainer) Setup(v *StorageVisibles) { + s.modules = v +} + +func NewStore(startingID int) (Storage, error) { + // TODO drop the startingID + + db, err := ConnectDB(DBSpec{ + Host: "postgres", + Port: 5432, + User: "exampleuser", + Password: "examplepass", + DBname: "querypack_db", + }) + if err != nil { + return nil, err + } + + s := StorageContainer{RequestID: startingID, DB: db} + if err := s.SetupDB(); err != nil { + return nil, err + } + + if err = s.loadState(); err != nil { + return nil, err + } + + return &s, nil +} + +func NewStorageContainer(startingID int) (*StorageContainer, error) { + + db, err := ConnectDB(DBSpec{ + Host: "postgres", + Port: 5432, + User: "exampleuser", + Password: "examplepass", + DBname: "server_db", + }) + if err != nil { + return nil, err + } + + s := StorageContainer{RequestID: startingID, DB: db} + if err := s.SetupDB(); err != nil { + return nil, err + } + + if err = s.loadState(); err != nil { + return nil, err + } + + return &s, nil +} + +func ConnectDB(s DBSpec) (*gorm.DB, error) { + // Open the database connection + dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", + s.Host, s.Port, s.User, s.Password, s.DBname) + db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{}) + if err != nil { + slog.Error("Error connecting to the database", "err", err) + return nil, err + } + return db, nil +} + +func (s *StorageContainer) loadState() error { + // TODO load the state + return nil +} diff --git a/pkg/qpstore/interfaces.go b/pkg/qpstore/interfaces.go new file mode 100644 index 0000000..26081fc --- /dev/null +++ b/pkg/qpstore/interfaces.go @@ -0,0 +1,13 @@ +package qpstore + +import ( + "mrvacommander/pkg/common" + "mrvacommander/pkg/qldbstore" +) + +type Storage interface { + NextID() int + SaveQueryPack(tgz []byte, sessionID int) (storagePath string, error error) + FindAvailableDBs(analysisReposRequested []common.OwnerRepo) (not_found_repos []common.OwnerRepo, + analysisRepos *map[common.OwnerRepo]qldbstore.DBLocation) +} diff --git a/pkg/server/types.go b/pkg/server/types.go index 0f221e9..7723c89 100644 --- a/pkg/server/types.go +++ b/pkg/server/types.go @@ -3,6 +3,7 @@ package server import ( "mrvacommander/pkg/common" "mrvacommander/pkg/logger" + "mrvacommander/pkg/qpstore" "mrvacommander/pkg/queue" "mrvacommander/pkg/storage" ) @@ -46,7 +47,7 @@ type CommanderVisibles struct { Queue queue.Queue ServerStore storage.Storage // TODO extra package for query pack storage - QueryPackStore storage.Storage + QueryPackStore qpstore.Storage // TODO extra package for ql db storage QLDBStore storage.Storage } diff --git a/pkg/storage/container.go b/pkg/storage/container.go index 65eb4ea..8a3febb 100644 --- a/pkg/storage/container.go +++ b/pkg/storage/container.go @@ -41,33 +41,6 @@ func NewQLDBStore() (*StorageContainer, error) { return nil, nil } -func NewQueryPackStore(startingID int) (*StorageContainer, error) { - // TODO set up querypack_db - // TODO drop the startingID - - db, err := ConnectDB(DBSpec{ - Host: "postgres", - Port: 5432, - User: "exampleuser", - Password: "examplepass", - DBname: "querypack_db", - }) - if err != nil { - return nil, err - } - - s := StorageContainer{RequestID: startingID, DB: db} - if err := s.SetupDB(); err != nil { - return nil, err - } - - if err = s.loadState(); err != nil { - return nil, err - } - - return &s, nil -} - func NewStorageContainer(startingID int) (*StorageContainer, error) { db, err := ConnectDB(DBSpec{ From 9c0cdb1fe43f062c639a64056b8aed5dca212615 Mon Sep 17 00:00:00 2001 From: Michael Hohn Date: Tue, 11 Jun 2024 16:55:10 -0700 Subject: [PATCH 4/7] Simplify naming, don't restate package name --- cmd/server/main.go | 20 ++++++++++---------- pkg/agent/agent.go | 4 ++-- pkg/logger/types.go | 6 +++--- pkg/queue/types.go | 6 +++--- pkg/server/server.go | 2 +- pkg/server/types.go | 4 ++-- pkg/storage/container.go | 2 +- pkg/storage/storage.go | 2 +- pkg/storage/types.go | 6 +++--- 9 files changed, 26 insertions(+), 26 deletions(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index 3eb2d0d..bc4ba5e 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -87,7 +87,7 @@ func main() { os.Exit(1) } - sc.Setup(&server.CommanderVisibles{ + sc.Setup(&server.Visibles{ Logger: sl, Queue: sq, ServerStore: ss, @@ -95,15 +95,15 @@ func main() { QLDBStore: ql, }) - sl.Setup(&logger.LoggerVisibles{}) + sl.Setup(&logger.Visibles{}) - sq.Setup(&queue.QueueVisibles{ + sq.Setup(&queue.Visibles{ Logger: sl, }) - ss.Setup(&storage.ServerStorageVisibles{}) + ss.Setup(&storage.Visibles{}) - sr.Setup(&agent.RunnerVisibles{ + sr.Setup(&agent.Visibles{ Logger: sl, Queue: sq, QueryPackStore: qp, @@ -136,7 +136,7 @@ func main() { sr := agent.NewRunnerSingle(2, sq) // FIXME take value from configuration - sc.Setup(&server.CommanderVisibles{ + sc.Setup(&server.Visibles{ Logger: sl, Queue: sq, ServerStore: ss, @@ -144,15 +144,15 @@ func main() { QLDBStore: ql, }) - sl.Setup(&logger.LoggerVisibles{}) + sl.Setup(&logger.Visibles{}) - sq.Setup(&queue.QueueVisibles{ + sq.Setup(&queue.Visibles{ Logger: sl, }) - ss.Setup(&storage.ServerStorageVisibles{}) + ss.Setup(&storage.Visibles{}) - sr.Setup(&agent.RunnerVisibles{ + sr.Setup(&agent.Visibles{ Logger: sl, Queue: sq, QueryPackStore: qp, diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 0592748..ef4dadd 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -33,7 +33,7 @@ func NewRunnerSingle(numWorkers int, queue queue.Queue) *RunnerSingle { return &r } -type RunnerVisibles struct { +type Visibles struct { Logger logger.Logger Queue queue.Queue // TODO extra package for query pack storage @@ -42,7 +42,7 @@ type RunnerVisibles struct { QLDBStore storage.Storage } -func (c *RunnerSingle) Setup(st *RunnerVisibles) { +func (c *RunnerSingle) Setup(st *Visibles) { return } diff --git a/pkg/logger/types.go b/pkg/logger/types.go index 7fe3bbd..dd08e48 100644 --- a/pkg/logger/types.go +++ b/pkg/logger/types.go @@ -1,7 +1,7 @@ package logger type LoggerSingle struct { - modules *LoggerVisibles + modules *Visibles } func NewLoggerSingle() *LoggerSingle { @@ -9,8 +9,8 @@ func NewLoggerSingle() *LoggerSingle { return &l } -type LoggerVisibles struct{} +type Visibles struct{} -func (l *LoggerSingle) Setup(v *LoggerVisibles) { +func (l *LoggerSingle) Setup(v *Visibles) { l.modules = v } diff --git a/pkg/queue/types.go b/pkg/queue/types.go index b8f5406..ad45c37 100644 --- a/pkg/queue/types.go +++ b/pkg/queue/types.go @@ -9,14 +9,14 @@ type QueueSingle struct { NumWorkers int jobs chan common.AnalyzeJob results chan common.AnalyzeResult - modules *QueueVisibles + modules *Visibles } -type QueueVisibles struct { +type Visibles struct { Logger logger.Logger } -func (q *QueueSingle) Setup(v *QueueVisibles) { +func (q *QueueSingle) Setup(v *Visibles) { q.modules = v } diff --git a/pkg/server/server.go b/pkg/server/server.go index 312af2b..603bb15 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -21,7 +21,7 @@ import ( "github.com/gorilla/mux" ) -func (c *CommanderSingle) Setup(st *CommanderVisibles) { +func (c *CommanderSingle) Setup(st *Visibles) { r := mux.NewRouter() c.st = st diff --git a/pkg/server/types.go b/pkg/server/types.go index 7723c89..927d2c4 100644 --- a/pkg/server/types.go +++ b/pkg/server/types.go @@ -26,7 +26,7 @@ type SessionInfo struct { } type CommanderSingle struct { - st *CommanderVisibles + st *Visibles } func NewCommanderSingle() *CommanderSingle { @@ -42,7 +42,7 @@ func NewCommanderSingle() *CommanderSingle { // Runner agent.Runner // } -type CommanderVisibles struct { +type Visibles struct { Logger logger.Logger Queue queue.Queue ServerStore storage.Storage diff --git a/pkg/storage/container.go b/pkg/storage/container.go index 8a3febb..3ba0214 100644 --- a/pkg/storage/container.go +++ b/pkg/storage/container.go @@ -32,7 +32,7 @@ func (s *StorageContainer) FindAvailableDBs(analysisReposRequested []common.Owne return notFoundRepos, analysisRepos } -func (s *StorageContainer) Setup(v *ServerStorageVisibles) { +func (s *StorageContainer) Setup(v *Visibles) { s.modules = v } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 1fe6dda..42f394e 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -28,7 +28,7 @@ func NewStorageSingle(startingID int) *StorageSingle { return &s } -func (s *StorageSingle) Setup(v *ServerStorageVisibles) { +func (s *StorageSingle) Setup(v *Visibles) { s.modules = v } diff --git a/pkg/storage/types.go b/pkg/storage/types.go index 7c94c02..c464669 100644 --- a/pkg/storage/types.go +++ b/pkg/storage/types.go @@ -13,7 +13,7 @@ type DBLocation struct { type StorageSingle struct { currentID int - modules *ServerStorageVisibles + modules *Visibles } type DBSpec struct { @@ -60,7 +60,7 @@ type StorageContainer struct { // Database version of StorageSingle RequestID int DB *gorm.DB - modules *ServerStorageVisibles + modules *Visibles } -type ServerStorageVisibles struct{} +type Visibles struct{} From 4b721b5969a8b46d44c2a26cb50d3618405c6fbc Mon Sep 17 00:00:00 2001 From: Michael Hohn Date: Tue, 11 Jun 2024 18:05:44 -0700 Subject: [PATCH 5/7] Fix docker-compose, add postgres init script --- README.md | 7 +++++-- docker-compose.yml | 5 +++-- postgres-init-scripts/dbinit.sh | 8 ++++++++ 3 files changed, 16 insertions(+), 4 deletions(-) create mode 100755 postgres-init-scripts/dbinit.sh diff --git a/README.md b/README.md index 2d1f3ce..502a8f6 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,10 @@ These are simple steps using a single container. ### Manually create needed postgres databases -This is still necessary after `docker-compose up` to avoid +docker-compose now runs a db init script, but this information is useful for +debugging/manual work. + +~~This is still necessary after `docker-compose up` to avoid~~ [error] failed to initialize database, got error failed to connect to `user=exampleuser database=server_db`: 172.25.0.3:5432 (postgres): server @@ -122,7 +125,7 @@ The steps: # List all dbs \l -To run pgmin, the minimal go/postgres test part of this repository: +### To run pgmin, the minimal go/postgres test part of this repository 1. Run pgmin diff --git a/docker-compose.yml b/docker-compose.yml index 3620c1a..55957eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,8 @@ services: POSTGRES_PASSWORD: examplepass POSTGRES_DB: exampledb volumes: - - postgres_data:/Users/hohn/var/lib/postgresql/data + - postgres_data:/var/lib/postgresql/data + - ./postgres-init-scripts:/docker-entrypoint-initdb.d ports: - "5432:5432" # Exposing PostgreSQL to the host expose: @@ -38,7 +39,7 @@ services: container_name: server command: sh -c "apt-get update && apt-get install -y curl && tail -f /dev/null" ports: - - "8080:80" # Exposing port 80 inside the container as port 8080 on the host + - "8080:8080" volumes: - /Users/hohn/work-gh/mrva/mrvacommander:/mrva/mrvacommander depends_on: diff --git a/postgres-init-scripts/dbinit.sh b/postgres-init-scripts/dbinit.sh new file mode 100755 index 0000000..37b058d --- /dev/null +++ b/postgres-init-scripts/dbinit.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + SELECT 'CREATE DATABASE server_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'server_db')\gexec + SELECT 'CREATE DATABASE querypack_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'querypack_db')\gexec + SELECT 'CREATE DATABASE qldb_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'qldb_db')\gexec +EOSQL From 765a76f75adab4b7c51e3a0340bac9684ef2b615 Mon Sep 17 00:00:00 2001 From: Michael Hohn Date: Tue, 11 Jun 2024 20:13:13 -0700 Subject: [PATCH 6/7] Provide MRVA_SERVER_ROOT via environment variable --- docker-compose.yml | 2 ++ pkg/agent/agent.go | 19 +++++++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 55957eb..df5297a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,6 +37,8 @@ services: server: image: ubuntu:22.04 container_name: server + environment: + - MRVA_SERVER_ROOT=/mrva/mrvacommander/cmd/server command: sh -c "apt-get update && apt-get install -y curl && tail -f /dev/null" ports: - "8080:8080" diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index ef4dadd..3a62fbe 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -82,21 +82,20 @@ func (r *RunnerSingle) RunAnalysis(job common.AnalyzeJob) (string, error) { queryPackID, dbOwner, dbRepo := job.QueryPackId, job.ORepo.Owner, job.ORepo.Repo - // FIXME Provide this via environment or explicit argument - gmsRoot := "/Users/hohn/work-gh/mrva/mrvacommander/cmd/server" + serverRoot := os.Getenv("MRVA_SERVER_ROOT") // Set up derived paths - dbPath := filepath.Join(gmsRoot, "var/codeql/dbs", dbOwner, dbRepo) - dbZip := filepath.Join(gmsRoot, "codeql/dbs", dbOwner, dbRepo, + dbPath := filepath.Join(serverRoot, "var/codeql/dbs", dbOwner, dbRepo) + dbZip := filepath.Join(serverRoot, "codeql/dbs", dbOwner, dbRepo, fmt.Sprintf("%s_%s_db.zip", dbOwner, dbRepo)) - dbExtract := filepath.Join(gmsRoot, "var/codeql/dbs", dbOwner, dbRepo) + dbExtract := filepath.Join(serverRoot, "var/codeql/dbs", dbOwner, dbRepo) - queryPack := filepath.Join(gmsRoot, + queryPack := filepath.Join(serverRoot, "var/codeql/querypacks", fmt.Sprintf("qp-%d.tgz", queryPackID)) - queryExtract := filepath.Join(gmsRoot, + queryExtract := filepath.Join(serverRoot, "var/codeql/querypacks", fmt.Sprintf("qp-%d", queryPackID)) - queryOutDir := filepath.Join(gmsRoot, + queryOutDir := filepath.Join(serverRoot, "var/codeql/sarif/localrun", dbOwner, dbRepo) queryOutFile := filepath.Join(queryOutDir, fmt.Sprintf("%s_%s.sarif", dbOwner, dbRepo)) @@ -108,7 +107,7 @@ func (r *RunnerSingle) RunAnalysis(job common.AnalyzeJob) (string, error) { } if err := unzipFile(dbZip, dbExtract); err != nil { - slog.Error("Failed to unzip DB %s: %v", dbZip, err) + slog.Error("Failed to unzip DB", dbZip, err) return "", err } @@ -133,7 +132,7 @@ func (r *RunnerSingle) RunAnalysis(job common.AnalyzeJob) (string, error) { cmd := exec.Command("codeql", "database", "analyze", "--format=sarif-latest", "--rerun", "--output", queryOutFile, "-j8", dbPath, queryExtract) - cmd.Dir = gmsRoot + cmd.Dir = serverRoot cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr From 5730c330f413176cd1e75f0b422059c8bdc708a8 Mon Sep 17 00:00:00 2001 From: Michael Hohn Date: Wed, 12 Jun 2024 11:28:37 -0700 Subject: [PATCH 7/7] Add codeql to server container for standalone testing For full test, we cannot have ERROR codeql database analyze failed: error="exec: \"codeql\": executable file not found in $PATH" job="{MirvaRequestID:0 QueryPackId:54674 QueryLanguage:cpp ORepo:{Owner:psycopg Repo:psycopg2}}" For linux/arm64, use a Dockerfile that: - uses ubuntu 22.04 base image - adds the 1.17 version of the codeql bundle - extracts the bundle - adds a recent version of the JRE - extracts it - sets the CODEQL_JAVA_HOME environment variable to point to the JRE The instructions are updated --- README.md | 3 +++ cmd/server/Dockerfile | 40 ++++++++++++++++++++++++++++++++-------- docker-compose.yml | 4 ++-- pkg/storage/storage.go | 3 +++ 4 files changed, 40 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 502a8f6..3e8f732 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ These are simple steps using a single container. 2. build docker image + cd cmd/server docker build -t server-image . 3. Start container with shared directory @@ -32,6 +33,8 @@ These are simple steps using a single container. ## Using docker-compose ### Steps to build and run the server in a multi-container environment set up by docker-compose. +1. Built the server-image, above + 1. Build server on host cd ~/work-gh/mrva/mrvacommander/cmd/server/ diff --git a/cmd/server/Dockerfile b/cmd/server/Dockerfile index 8780222..d12ebc8 100644 --- a/cmd/server/Dockerfile +++ b/cmd/server/Dockerfile @@ -1,14 +1,38 @@ -# Use Ubuntu 22.04 as the base image -FROM arm64v8/ubuntu:22.04 +# Use the ubuntu 22.04 base image +FROM ubuntu:22.04 -# Set environment variables to non-interactive to avoid prompts during installation +# Set architecture to arm64 +ARG ARCH=arm64 +ARG AARCH=aarch64 + +# Set environment variables ENV DEBIAN_FRONTEND=noninteractive +ENV CODEQL_VERSION=codeql-bundle-v2.17.5 +ENV CODEQL_DOWNLOAD_URL=https://github.com/github/codeql-action/releases/download/${CODEQL_VERSION}/codeql-bundle-linux64.tar.gz +ENV JDK_VERSION=22.0.1 +ENV JDK_DOWNLOAD_URL=https://download.oracle.com/java/21/latest/jdk-${JDK_VERSION}_linux-${AARCH}_bin.tar.gz +ENV JDK_DOWNLOAD_URL=https://download.java.net/java/GA/jdk${JDK_VERSION}/c7ec1332f7bb44aeba2eb341ae18aca4/8/GPL/openjdk-${JDK_VERSION}_linux-${AARCH}_bin.tar.gz -# Update the package list +ENV CODEQL_JAVA_HOME=/usr/local/jdk-${JDK_VERSION} + +# Install necessary tools RUN apt-get update && \ - apt-get clean + apt-get install -y curl tar && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* +# Add and extract the CodeQL bundle +RUN curl -L $CODEQL_DOWNLOAD_URL -o /tmp/${CODEQL_VERSION}.tar.gz && \ + tar -xzf /tmp/${CODEQL_VERSION}.tar.gz -C /opt && \ + rm /tmp/${CODEQL_VERSION}.tar.gz + +# Add and extract the JDK +RUN curl -L $JDK_DOWNLOAD_URL -o /tmp/jdk-${JDK_VERSION}.tar.gz && \ + tar -xzf /tmp/jdk-${JDK_VERSION}.tar.gz -C /usr/local && \ + rm /tmp/jdk-${JDK_VERSION}.tar.gz + +# Set PATH +ENV PATH=/opt/codeql:"$PATH" + +# Prepare host mount point RUN mkdir /mrva - -# Set the default command -CMD ["bash"] diff --git a/docker-compose.yml b/docker-compose.yml index df5297a..e8a5e03 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,11 +35,11 @@ services: server: - image: ubuntu:22.04 + image: server-image container_name: server environment: - MRVA_SERVER_ROOT=/mrva/mrvacommander/cmd/server - command: sh -c "apt-get update && apt-get install -y curl && tail -f /dev/null" + command: sh -c "tail -f /dev/null" ports: - "8080:8080" volumes: diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 42f394e..cfb19f9 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -115,6 +115,9 @@ func ArtifactURL(js common.JobSpec, vaid int) (string, error) { slog.Error("Error packaging results:", "error", err) return "", err } + // TODO Need url valid in container network and externally + // For now, we assume the container port 8080 is port 8080 on user's machine + hostname = "localhost" au := fmt.Sprintf("http://%s:8080/download-server/%s", hostname, zfpath) return au, nil }