mrva/mrvacommander
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip: Make cross-module visibility explicit via Visibles structs

Browse Source 
All access is/will be through interfaces accessed through these structs.

This introduces several distinct storage units:
+ DB for server state
+ DB for codeql databases
+ query pack store

The steps for manually creating needed databases are in the README
This commit is contained in:
Michael Hohn
2024-06-07 13:14:41 -07:00
committed by =Michael Hohn
parent 25cab583c1
commit 7e0d6909da
10 changed files with 205 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
README.md
View File

@@ -86,6 +86,33 @@ Some postgres specific commands
\dt
1. Examine a table
select * from db_infos
1. Show all columns in a specific table
\d+ db_infos
1. Miscellany
\pset pager off
\lo_import FILE [COMMENT]
Manually create needed postgres databases
# on the host
psql -h localhost -p 5432 -U exampleuser -d postgres
# Conditionally create dbs
SELECT 'CREATE DATABASE server_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'server_db')\gexec
SELECT 'CREATE DATABASE querypack_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'querypack_db')\gexec
SELECT 'CREATE DATABASE qldb_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'qldb_db')\gexec
# List all dbs
\l
To run pgmin, the minimal go/postgres test part of this repository:
1. Run pgmin

83
cmd/server/main.go
View File

@@ -74,38 +74,89 @@ func main() {
ss := storage.NewStorageSingle(config.Storage.StartingID)
sr := agent.NewRunnerSingle(2, sq) // FIXME take value from configuration
state := server.State{
Commander: sc,
Logger: sl,
Queue: sq,
Storage: ss,
Runner: sr,
qp, err := storage.NewQueryPackStore(config.Storage.StartingID)
if err != nil {
slog.Error("Unable to initialize query pack storage")
os.Exit(1)
}
sc.Setup(&state) // sc is part of state and dereferences it
ql, err := storage.NewQLDBStore()
if err != nil {
slog.Error("Unable to initialize ql database storage")
os.Exit(1)
}
sc.Setup(&server.CommanderVisibles{
Logger: sl,
Queue: sq,
ServerStore: ss,
QueryPackStore: qp,
QLDBStore: ql,
})
sl.Setup(&logger.LoggerVisibles{})
sq.Setup(&queue.QueueVisibles{
Logger: sl,
})
ss.Setup(&storage.ServerStorageVisibles{})
sr.Setup(&agent.RunnerVisibles{
Logger: sl,
Queue: sq,
QueryPackStore: qp,
QLDBStore: ql,
})
case "container":
// Assemble container version
sq := queue.NewQueueSingle(2) // FIXME take value from configuration
sc := server.NewCommanderSingle(nil, sq)
sl := logger.NewLoggerSingle()
ss, err := storage.NewStorageContainer(config.Storage.StartingID)
if err != nil {
slog.Error("Unable to initialize storage")
slog.Error("Unable to initialize server storage")
os.Exit(1)
}
qp, err := storage.NewQueryPackStore(config.Storage.StartingID)
if err != nil {
slog.Error("Unable to initialize query pack storage")
os.Exit(1)
}
ql, err := storage.NewQLDBStore()
if err != nil {
slog.Error("Unable to initialize ql database storage")
os.Exit(1)
}
sr := agent.NewRunnerSingle(2, sq) // FIXME take value from configuration
state := server.State{
Commander: sc,
Logger: sl,
Queue: sq,
Storage: ss,
Runner: sr,
}
sc.Setup(&server.CommanderVisibles{
Logger: sl,
Queue: sq,
ServerStore: ss,
QueryPackStore: qp,
QLDBStore: ql,
})
sc.Setup(&state) // sc is part of state and dereferences it
sl.Setup(&logger.LoggerVisibles{})
sq.Setup(&queue.QueueVisibles{
Logger: sl,
})
ss.Setup(&storage.ServerStorageVisibles{})
sr.Setup(&agent.RunnerVisibles{
Logger: sl,
Queue: sq,
QueryPackStore: qp,
QLDBStore: ql,
})
case "cluster":
// Assemble cccluster

14
pkg/agent/agent.go
View File

@@ -2,6 +2,7 @@ package agent
import (
"mrvacommander/pkg/common"
"mrvacommander/pkg/logger"
"mrvacommander/pkg/queue"
"mrvacommander/pkg/storage"
@@ -31,6 +32,19 @@ func NewRunnerSingle(numWorkers int, queue queue.Queue) *RunnerSingle {
return &r
}
type RunnerVisibles struct {
Logger logger.Logger
Queue queue.Queue
// TODO extra package for query pack storage
QueryPackStore storage.Storage
// TODO extra package for ql db storage
QLDBStore storage.Storage
}
func (c *RunnerSingle) Setup(st *RunnerVisibles) {
return
}
func (r *RunnerSingle) worker(wid int) {
var job common.AnalyzeJob

7
pkg/logger/types.go
View File

@@ -1,9 +1,16 @@
package logger
type LoggerSingle struct {
modules *LoggerVisibles
}
func NewLoggerSingle() *LoggerSingle {
l := LoggerSingle{}
return &l
}
type LoggerVisibles struct{}
func (l *LoggerSingle) Setup(v *LoggerVisibles) {
l.modules = v
}

14
pkg/queue/types.go
View File

@@ -1,11 +1,23 @@
package queue
import "mrvacommander/pkg/common"
import (
"mrvacommander/pkg/common"
"mrvacommander/pkg/logger"
)
type QueueSingle struct {
NumWorkers int
jobs chan common.AnalyzeJob
results chan common.AnalyzeResult
modules *QueueVisibles
}
type QueueVisibles struct {
Logger logger.Logger
}
func (q *QueueSingle) Setup(v *QueueVisibles) {
q.modules = v
}
func NewQueueSingle(numWorkers int) *QueueSingle {

8
pkg/server/server.go
View File

@@ -21,7 +21,7 @@ import (
"github.com/gorilla/mux"
)
func (c *CommanderSingle) Setup(st *State) {
func (c *CommanderSingle) Setup(st *CommanderVisibles) {
r := mux.NewRouter()
c.st = st
@@ -275,7 +275,7 @@ func (c *CommanderSingle) MirvaRequest(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
slog.Info("New mrva run ", "owner", vars["owner"], "repo", vars["repo"])
session_id := c.st.Storage.NextID()
session_id := c.st.ServerStore.NextID()
session_owner := vars["owner"]
session_controller_repo := vars["repo"]
slog.Info("new run", "id: ", fmt.Sprint(session_id), session_owner, session_controller_repo)
@@ -284,7 +284,7 @@ func (c *CommanderSingle) MirvaRequest(w http.ResponseWriter, r *http.Request) {
return
}
not_found_repos, analysisRepos := c.st.Storage.FindAvailableDBs(session_repositories)
not_found_repos, analysisRepos := c.st.ServerStore.FindAvailableDBs(session_repositories)
c.queue.StartAnalyses(analysisRepos, session_id, session_language)
@@ -492,7 +492,7 @@ func (c *CommanderSingle) extract_tgz(qp string, sessionID int) (string, error)
return "", err
}
session_query_pack_tgz_filepath, err := c.st.Storage.SaveQueryPack(tgz, sessionID)
session_query_pack_tgz_filepath, err := c.st.ServerStore.SaveQueryPack(tgz, sessionID)
if err != nil {
return "", err
}

28
pkg/server/types.go
View File

@@ -1,7 +1,6 @@
package server
import (
"mrvacommander/pkg/agent"
"mrvacommander/pkg/common"
"mrvacommander/pkg/logger"
"mrvacommander/pkg/queue"
@@ -26,19 +25,30 @@ type SessionInfo struct {
}
type CommanderSingle struct {
st *State
st *CommanderVisibles
// TODO remove:
queue queue.Queue
}
func NewCommanderSingle(s *State, q queue.Queue) *CommanderSingle {
func NewCommanderSingle(s *CommanderVisibles, q queue.Queue) *CommanderSingle {
c := CommanderSingle{s, q}
return &c
}
type State struct {
Commander Commander
Logger logger.Logger
Queue queue.Queue
Storage storage.Storage
Runner agent.Runner
// type State struct {
// Commander Commander
// Logger logger.Logger
// Queue queue.Queue
// Storage storage.Storage
// Runner agent.Runner
// }
type CommanderVisibles struct {
Logger logger.Logger
Queue queue.Queue
ServerStore storage.Storage
// TODO extra package for query pack storage
QueryPackStore storage.Storage
// TODO extra package for ql db storage
QLDBStore storage.Storage
}

72
pkg/storage/container.go
View File

@@ -32,6 +32,42 @@ func (s *StorageContainer) FindAvailableDBs(analysisReposRequested []common.Owne
return notFoundRepos, analysisRepos
}
func (s *StorageContainer) Setup(v *ServerStorageVisibles) {
s.modules = v
}
func NewQLDBStore() (*StorageContainer, error) {
// TODO set up qldb_db
return nil, nil
}
func NewQueryPackStore(startingID int) (*StorageContainer, error) {
// TODO set up querypack_db
// TODO drop the startingID
db, err := ConnectDB(DBSpec{
Host: "postgres",
Port: 5432,
User: "exampleuser",
Password: "examplepass",
DBname: "querypack_db",
})
if err != nil {
return nil, err
}
s := StorageContainer{RequestID: startingID, DB: db}
if err := s.SetupDB(); err != nil {
return nil, err
}
if err = s.loadState(); err != nil {
return nil, err
}
return &s, nil
}
func NewStorageContainer(startingID int) (*StorageContainer, error) {
db, err := ConnectDB(DBSpec{
@@ -39,32 +75,21 @@ func NewStorageContainer(startingID int) (*StorageContainer, error) {
Port: 5432,
User: "exampleuser",
Password: "examplepass",
DBname: "exampledb",
DBname: "server_db",
})
if err != nil {
return nil, err
}
s, err := LoadOrInit(db, startingID)
if err != nil {
return nil, err
}
return s, nil
}
func LoadOrInit(db *gorm.DB, startingID int) (*StorageContainer, error) {
// Check and set up the database
s := StorageContainer{RequestID: startingID, DB: db}
if s.hasTables() {
s.loadState()
} else {
if err := s.SetupDB(); err != nil {
return nil, err
}
s.setFresh()
if err := s.SetupDB(); err != nil {
return nil, err
}
if err = s.loadState(); err != nil {
return nil, err
}
return &s, nil
}
@@ -80,12 +105,7 @@ func ConnectDB(s DBSpec) (*gorm.DB, error) {
return db, nil
}
func (s *StorageContainer) setFresh() {
// TODO Set initial state
}
func (s *StorageContainer) SetupDB() error {
// TODO Migrate the schemas
msg := "Failed to initialize database "
if err := s.DB.AutoMigrate(&DBInfo{}); err != nil {
@@ -108,9 +128,9 @@ func (s *StorageContainer) SetupDB() error {
return nil
}
func (s *StorageContainer) loadState() {
func (s *StorageContainer) loadState() error {
// TODO load the state
return
return nil
}
func (s *StorageContainer) hasTables() bool {

4
pkg/storage/storage.go
View File

@@ -28,6 +28,10 @@ func NewStorageSingle(startingID int) *StorageSingle {
return &s
}
func (s *StorageSingle) Setup(v *ServerStorageVisibles) {
s.modules = v
}
func (s *StorageSingle) NextID() int {
s.currentID += 1
return s.currentID

4
pkg/storage/types.go
View File

@@ -13,6 +13,7 @@ type DBLocation struct {
type StorageSingle struct {
currentID int
modules *ServerStorageVisibles
}
type DBSpec struct {
@@ -59,4 +60,7 @@ type StorageContainer struct {
// Database version of StorageSingle
RequestID int
DB *gorm.DB
modules *ServerStorageVisibles
}
type ServerStorageVisibles struct{}