Merge pull request #12 from advanced-security/hohn-0.1.11

Hohn 0.1.11  Make cross-module visibility explicit

README.md

@@ -15,6 +15,7 @@ These are simple steps using a single container.
 
 2.  build docker image
 
+        cd cmd/server
         docker build -t server-image .
 
 3.  Start container with shared directory
@@ -30,7 +31,9 @@ These are simple steps using a single container.
         cd /mrva/mrvacommander/cmd/server/ && ./server
 
 ## Using docker-compose
-Steps to build and run the server in a multi-container environment set up by docker-compose.
+### Steps to build and run the server in a multi-container environment set up by docker-compose.
+
+1.  Built the server-image, above
 
 1.  Build server on host
 
@@ -51,11 +54,8 @@ Steps to build and run the server in a multi-container environment set up by doc
         ./server -loglevel=debug -mode=container
 
 1.  Test server via remote client by following the steps in [gh-mrva](https://github.com/hohn/gh-mrva/blob/connection-redirect/README.org#compacted-edit-run-debug-cycle)
-    
 
-
+### Some general docker-compose commands
-
-Some general docker-compose commands
 
 2.  Get service status
 
@@ -76,7 +76,7 @@ Some general docker-compose commands
         curl -I http://rabbitmq:15672
 
 
-Some postgres specific commands
+### Some postgres specific commands
 
 1.  Access PostgreSQL
     
@@ -86,7 +86,49 @@ Some postgres specific commands
     
         \dt
 
-To run pgmin, the minimal go/postgres test part of this repository:
+1.  Examine a table
+
+        select * from db_infos
+
+1.  Show all columns in a specific table
+
+        \d+ db_infos
+
+1.  Miscellany
+
+        \pset pager off
+        \lo_import FILE [COMMENT]
+
+
+### Manually create needed postgres databases
+
+docker-compose now runs a db init script, but this information is useful for 
+debugging/manual work.
+
+~~This is still necessary after `docker-compose up` to avoid~~
+
+    [error] failed to initialize database, got error failed to connect to
+    `user=exampleuser database=server_db`: 172.25.0.3:5432 (postgres): server
+    error: FATAL: database "server_db" does not exist (SQLSTATE 3D000) 
+
+from
+
+    ./server -loglevel=debug -mode=container
+
+The steps:
+
+    # on the host
+    psql -h localhost -p 5432 -U exampleuser -d postgres
+
+    # Conditionally create dbs
+    SELECT 'CREATE DATABASE server_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'server_db')\gexec
+    SELECT 'CREATE DATABASE querypack_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'querypack_db')\gexec
+    SELECT 'CREATE DATABASE qldb_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'qldb_db')\gexec
+
+    # List all dbs
+    \l
+
+### To run pgmin, the minimal go/postgres test part of this repository
 
 1.  Run pgmin
 

cmd/server/Dockerfile

@@ -1,14 +1,38 @@
-# Use Ubuntu 22.04 as the base image
+# Use the ubuntu 22.04 base image
-FROM arm64v8/ubuntu:22.04
+FROM ubuntu:22.04
 
-# Set environment variables to non-interactive to avoid prompts during installation
+# Set architecture to arm64
+ARG ARCH=arm64
+ARG AARCH=aarch64
+
+# Set environment variables
 ENV DEBIAN_FRONTEND=noninteractive
+ENV CODEQL_VERSION=codeql-bundle-v2.17.5
+ENV CODEQL_DOWNLOAD_URL=https://github.com/github/codeql-action/releases/download/${CODEQL_VERSION}/codeql-bundle-linux64.tar.gz
+ENV JDK_VERSION=22.0.1
+ENV JDK_DOWNLOAD_URL=https://download.oracle.com/java/21/latest/jdk-${JDK_VERSION}_linux-${AARCH}_bin.tar.gz
+ENV JDK_DOWNLOAD_URL=https://download.java.net/java/GA/jdk${JDK_VERSION}/c7ec1332f7bb44aeba2eb341ae18aca4/8/GPL/openjdk-${JDK_VERSION}_linux-${AARCH}_bin.tar.gz
 
-# Update the package list
+ENV CODEQL_JAVA_HOME=/usr/local/jdk-${JDK_VERSION}
+
+# Install necessary tools
 RUN apt-get update && \
-    apt-get clean 
+    apt-get install -y curl tar && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
+# Add and extract the CodeQL bundle
+RUN curl -L $CODEQL_DOWNLOAD_URL -o /tmp/${CODEQL_VERSION}.tar.gz && \
+    tar -xzf /tmp/${CODEQL_VERSION}.tar.gz -C /opt && \
+    rm /tmp/${CODEQL_VERSION}.tar.gz
+
+# Add and extract the JDK
+RUN curl -L $JDK_DOWNLOAD_URL -o /tmp/jdk-${JDK_VERSION}.tar.gz && \
+    tar -xzf /tmp/jdk-${JDK_VERSION}.tar.gz -C /usr/local && \
+    rm /tmp/jdk-${JDK_VERSION}.tar.gz
+
+# Set PATH
+ENV PATH=/opt/codeql:"$PATH"
+
+# Prepare host mount point
 RUN mkdir /mrva
-
-# Set the default command
-CMD ["bash"]

cmd/server/main.go

@@ -13,6 +13,7 @@ import (
 
 	"mrvacommander/pkg/agent"
 	"mrvacommander/pkg/logger"
+	"mrvacommander/pkg/qpstore"
 	"mrvacommander/pkg/queue"
 	"mrvacommander/pkg/server"
 	"mrvacommander/pkg/storage"
@@ -69,43 +70,94 @@ func main() {
 	case "standalone":
 		// Assemble single-process version
 		sq := queue.NewQueueSingle(2) // FIXME take value from configuration
-		sc := server.NewCommanderSingle(nil, sq)
+		sc := server.NewCommanderSingle()
 		sl := logger.NewLoggerSingle()
 		ss := storage.NewStorageSingle(config.Storage.StartingID)
 		sr := agent.NewRunnerSingle(2, sq) // FIXME take value from configuration
 
-		state := server.State{
+		qp, err := qpstore.NewStore(config.Storage.StartingID)
-			Commander: sc,
+		if err != nil {
-			Logger:    sl,
+			slog.Error("Unable to initialize query pack storage")
-			Queue:     sq,
+			os.Exit(1)
-			Storage:   ss,
-			Runner:    sr,
 		}
 
-		sc.Setup(&state) // sc is part of state and dereferences it
+		ql, err := storage.NewQLDBStore()
+		if err != nil {
+			slog.Error("Unable to initialize ql database storage")
+			os.Exit(1)
+		}
+
+		sc.Setup(&server.Visibles{
+			Logger:         sl,
+			Queue:          sq,
+			ServerStore:    ss,
+			QueryPackStore: qp,
+			QLDBStore:      ql,
+		})
+
+		sl.Setup(&logger.Visibles{})
+
+		sq.Setup(&queue.Visibles{
+			Logger: sl,
+		})
+
+		ss.Setup(&storage.Visibles{})
+
+		sr.Setup(&agent.Visibles{
+			Logger:         sl,
+			Queue:          sq,
+			QueryPackStore: qp,
+			QLDBStore:      ql,
+		})
 
 	case "container":
 		// Assemble container version
 		sq := queue.NewQueueSingle(2) // FIXME take value from configuration
-		sc := server.NewCommanderSingle(nil, sq)
+		sc := server.NewCommanderSingle()
 		sl := logger.NewLoggerSingle()
+
 		ss, err := storage.NewStorageContainer(config.Storage.StartingID)
 		if err != nil {
-			slog.Error("Unable to initialize storage")
+			slog.Error("Unable to initialize server storage")
+			os.Exit(1)
+		}
+
+		qp, err := qpstore.NewStore(config.Storage.StartingID)
+		if err != nil {
+			slog.Error("Unable to initialize query pack storage")
+			os.Exit(1)
+		}
+
+		ql, err := storage.NewQLDBStore()
+		if err != nil {
+			slog.Error("Unable to initialize ql database storage")
 			os.Exit(1)
 		}
 
 		sr := agent.NewRunnerSingle(2, sq) // FIXME take value from configuration
 
-		state := server.State{
+		sc.Setup(&server.Visibles{
-			Commander: sc,
+			Logger:         sl,
-			Logger:    sl,
+			Queue:          sq,
-			Queue:     sq,
+			ServerStore:    ss,
-			Storage:   ss,
+			QueryPackStore: qp,
-			Runner:    sr,
+			QLDBStore:      ql,
-		}
+		})
 
-		sc.Setup(&state) // sc is part of state and dereferences it
+		sl.Setup(&logger.Visibles{})
+
+		sq.Setup(&queue.Visibles{
+			Logger: sl,
+		})
+
+		ss.Setup(&storage.Visibles{})
+
+		sr.Setup(&agent.Visibles{
+			Logger:         sl,
+			Queue:          sq,
+			QueryPackStore: qp,
+			QLDBStore:      ql,
+		})
 
 	case "cluster":
 		// Assemble cccluster

docker-compose.yml

@@ -9,7 +9,8 @@ services:
       POSTGRES_PASSWORD: examplepass
       POSTGRES_DB: exampledb
     volumes:
-      - postgres_data:/Users/hohn/var/lib/postgresql/data
+      - postgres_data:/var/lib/postgresql/data
+      - ./postgres-init-scripts:/docker-entrypoint-initdb.d
     ports:
       - "5432:5432" # Exposing PostgreSQL to the host
     expose:
@@ -34,11 +35,13 @@ services:
 
 
   server:
-    image: ubuntu:22.04
+    image: server-image
     container_name: server
-    command: sh -c "apt-get update && apt-get install -y curl && tail -f /dev/null"
+    environment:
+      - MRVA_SERVER_ROOT=/mrva/mrvacommander/cmd/server
+    command: sh -c "tail -f /dev/null"
     ports:
-      - "8080:80" # Exposing port 80 inside the container as port 8080 on the host
+      - "8080:8080" 
     volumes:
       - /Users/hohn/work-gh/mrva/mrvacommander:/mrva/mrvacommander
     depends_on:
@@ -47,10 +50,28 @@ services:
     networks:
       - backend
 
+  minio:
+    image: minio/minio:RELEASE.2024-06-11T03-13-30Z
+    container_name: minio
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    environment:
+      MINIO_ROOT_USER: user
+      MINIO_ROOT_PASSWORD: musty-coke
+    command: server /data --console-address ":9001"
+    volumes:
+      - minio-data:/data
+
+volumes:
+  minio-data:
+  postgres_data:
+    driver: local
+
 networks:
   backend:
     driver: bridge
     
-volumes:
+
-  postgres_data:
+
-    driver: local
+

pkg/agent/agent.go

@@ -2,6 +2,8 @@ package agent
 
 import (
 	"mrvacommander/pkg/common"
+	"mrvacommander/pkg/logger"
+	"mrvacommander/pkg/qpstore"
 	"mrvacommander/pkg/queue"
 	"mrvacommander/pkg/storage"
 
@@ -31,6 +33,19 @@ func NewRunnerSingle(numWorkers int, queue queue.Queue) *RunnerSingle {
 	return &r
 }
 
+type Visibles struct {
+	Logger logger.Logger
+	Queue  queue.Queue
+	// TODO extra package for query pack storage
+	QueryPackStore qpstore.Storage
+	// TODO extra package for ql db storage
+	QLDBStore storage.Storage
+}
+
+func (c *RunnerSingle) Setup(st *Visibles) {
+	return
+}
+
 func (r *RunnerSingle) worker(wid int) {
 	var job common.AnalyzeJob
 
@@ -67,21 +82,20 @@ func (r *RunnerSingle) RunAnalysis(job common.AnalyzeJob) (string, error) {
 	queryPackID, dbOwner, dbRepo :=
 		job.QueryPackId, job.ORepo.Owner, job.ORepo.Repo
 
-	// FIXME Provide this via environment or explicit argument
+	serverRoot := os.Getenv("MRVA_SERVER_ROOT")
-	gmsRoot := "/Users/hohn/work-gh/mrva/mrvacommander/cmd/server"
 
 	// Set up derived paths
-	dbPath := filepath.Join(gmsRoot, "var/codeql/dbs", dbOwner, dbRepo)
+	dbPath := filepath.Join(serverRoot, "var/codeql/dbs", dbOwner, dbRepo)
-	dbZip := filepath.Join(gmsRoot, "codeql/dbs", dbOwner, dbRepo,
+	dbZip := filepath.Join(serverRoot, "codeql/dbs", dbOwner, dbRepo,
 		fmt.Sprintf("%s_%s_db.zip", dbOwner, dbRepo))
-	dbExtract := filepath.Join(gmsRoot, "var/codeql/dbs", dbOwner, dbRepo)
+	dbExtract := filepath.Join(serverRoot, "var/codeql/dbs", dbOwner, dbRepo)
 
-	queryPack := filepath.Join(gmsRoot,
+	queryPack := filepath.Join(serverRoot,
 		"var/codeql/querypacks", fmt.Sprintf("qp-%d.tgz", queryPackID))
-	queryExtract := filepath.Join(gmsRoot,
+	queryExtract := filepath.Join(serverRoot,
 		"var/codeql/querypacks", fmt.Sprintf("qp-%d", queryPackID))
 
-	queryOutDir := filepath.Join(gmsRoot,
+	queryOutDir := filepath.Join(serverRoot,
 		"var/codeql/sarif/localrun", dbOwner, dbRepo)
 	queryOutFile := filepath.Join(queryOutDir,
 		fmt.Sprintf("%s_%s.sarif", dbOwner, dbRepo))
@@ -93,7 +107,7 @@ func (r *RunnerSingle) RunAnalysis(job common.AnalyzeJob) (string, error) {
 	}
 
 	if err := unzipFile(dbZip, dbExtract); err != nil {
-		slog.Error("Failed to unzip DB %s: %v", dbZip, err)
+		slog.Error("Failed to unzip DB", dbZip, err)
 		return "", err
 	}
 
@@ -118,7 +132,7 @@ func (r *RunnerSingle) RunAnalysis(job common.AnalyzeJob) (string, error) {
 	cmd := exec.Command("codeql", "database", "analyze",
 		"--format=sarif-latest", "--rerun", "--output", queryOutFile,
 		"-j8", dbPath, queryExtract)
-	cmd.Dir = gmsRoot
+	cmd.Dir = serverRoot
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 

pkg/logger/types.go

@@ -1,9 +1,16 @@
 package logger
 
 type LoggerSingle struct {
+	modules *Visibles
 }
 
 func NewLoggerSingle() *LoggerSingle {
 	l := LoggerSingle{}
 	return &l
 }
+
+type Visibles struct{}
+
+func (l *LoggerSingle) Setup(v *Visibles) {
+	l.modules = v
+}

pkg/qldbstore/interfaces.go

@@ -0,0 +1,6 @@
+package qldbstore
+
+type DBLocation struct {
+	Prefix string
+	File   string
+}

pkg/qpstore/container.go

@@ -0,0 +1,138 @@
+package qpstore
+
+import (
+	"fmt"
+	"log/slog"
+	"mrvacommander/pkg/common"
+	"mrvacommander/pkg/qldbstore"
+	"sync"
+
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+)
+
+var (
+	DBmutex sync.Mutex
+)
+
+type StorageVisibles struct{}
+
+type StorageContainer struct {
+	// Database version of StorageSingle
+	RequestID int
+	DB        *gorm.DB
+	modules   *StorageVisibles
+}
+
+type DBSpec struct {
+	Host     string
+	Port     int
+	User     string
+	Password string
+	DBname   string
+}
+
+func (s *StorageContainer) SetupDB() error {
+	// TODO set up query pack storage
+	return nil
+}
+
+func (s *StorageContainer) LoadState() error {
+	// TODO load the state
+	return nil
+}
+
+func (s *StorageContainer) hasTables() bool {
+	// TODO query to check for tables
+	return false
+}
+
+func (s *StorageContainer) NextID() int {
+	// TODO update via db
+	return 12345
+}
+
+func (s *StorageContainer) SaveQueryPack(tgz []byte, sessionID int) (storagePath string, error error) {
+	// TODO save and return path
+	return "todo:no-path-yet", nil
+}
+
+func (s *StorageContainer) FindAvailableDBs(analysisReposRequested []common.OwnerRepo) (notFoundRepos []common.OwnerRepo, analysisRepos *map[common.OwnerRepo]qldbstore.DBLocation) {
+	// TODO  s.FindAvailableDBs() via postgres
+	analysisRepos = &map[common.OwnerRepo]qldbstore.DBLocation{}
+	notFoundRepos = []common.OwnerRepo{}
+
+	return notFoundRepos, analysisRepos
+}
+
+func (s *StorageContainer) Setup(v *StorageVisibles) {
+	s.modules = v
+}
+
+func NewStore(startingID int) (Storage, error) {
+	// TODO drop the startingID
+
+	db, err := ConnectDB(DBSpec{
+		Host:     "postgres",
+		Port:     5432,
+		User:     "exampleuser",
+		Password: "examplepass",
+		DBname:   "querypack_db",
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	s := StorageContainer{RequestID: startingID, DB: db}
+	if err := s.SetupDB(); err != nil {
+		return nil, err
+	}
+
+	if err = s.loadState(); err != nil {
+		return nil, err
+	}
+
+	return &s, nil
+}
+
+func NewStorageContainer(startingID int) (*StorageContainer, error) {
+
+	db, err := ConnectDB(DBSpec{
+		Host:     "postgres",
+		Port:     5432,
+		User:     "exampleuser",
+		Password: "examplepass",
+		DBname:   "server_db",
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	s := StorageContainer{RequestID: startingID, DB: db}
+	if err := s.SetupDB(); err != nil {
+		return nil, err
+	}
+
+	if err = s.loadState(); err != nil {
+		return nil, err
+	}
+
+	return &s, nil
+}
+
+func ConnectDB(s DBSpec) (*gorm.DB, error) {
+	// Open the database connection
+	dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
+		s.Host, s.Port, s.User, s.Password, s.DBname)
+	db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{})
+	if err != nil {
+		slog.Error("Error connecting to the database", "err", err)
+		return nil, err
+	}
+	return db, nil
+}
+
+func (s *StorageContainer) loadState() error {
+	// TODO load the state
+	return nil
+}

pkg/qpstore/interfaces.go

@@ -0,0 +1,13 @@
+package qpstore
+
+import (
+	"mrvacommander/pkg/common"
+	"mrvacommander/pkg/qldbstore"
+)
+
+type Storage interface {
+	NextID() int
+	SaveQueryPack(tgz []byte, sessionID int) (storagePath string, error error)
+	FindAvailableDBs(analysisReposRequested []common.OwnerRepo) (not_found_repos []common.OwnerRepo,
+		analysisRepos *map[common.OwnerRepo]qldbstore.DBLocation)
+}

pkg/queue/types.go

@@ -1,11 +1,23 @@
 package queue
 
-import "mrvacommander/pkg/common"
+import (
+	"mrvacommander/pkg/common"
+	"mrvacommander/pkg/logger"
+)
 
 type QueueSingle struct {
 	NumWorkers int
 	jobs       chan common.AnalyzeJob
 	results    chan common.AnalyzeResult
+	modules    *Visibles
+}
+
+type Visibles struct {
+	Logger logger.Logger
+}
+
+func (q *QueueSingle) Setup(v *Visibles) {
+	q.modules = v
 }
 
 func NewQueueSingle(numWorkers int) *QueueSingle {

pkg/server/server.go

@@ -21,7 +21,7 @@ import (
 	"github.com/gorilla/mux"
 )
 
-func (c *CommanderSingle) Setup(st *State) {
+func (c *CommanderSingle) Setup(st *Visibles) {
 	r := mux.NewRouter()
 	c.st = st
 
@@ -275,7 +275,7 @@ func (c *CommanderSingle) MirvaRequest(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	slog.Info("New mrva run ", "owner", vars["owner"], "repo", vars["repo"])
 
-	session_id := c.st.Storage.NextID()
+	session_id := c.st.ServerStore.NextID()
 	session_owner := vars["owner"]
 	session_controller_repo := vars["repo"]
 	slog.Info("new run", "id: ", fmt.Sprint(session_id), session_owner, session_controller_repo)
@@ -284,9 +284,9 @@ func (c *CommanderSingle) MirvaRequest(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	not_found_repos, analysisRepos := c.st.Storage.FindAvailableDBs(session_repositories)
+	not_found_repos, analysisRepos := c.st.ServerStore.FindAvailableDBs(session_repositories)
 
-	c.queue.StartAnalyses(analysisRepos, session_id, session_language)
+	c.st.Queue.StartAnalyses(analysisRepos, session_id, session_language)
 
 	si := SessionInfo{
 		ID:             session_id,
@@ -492,7 +492,7 @@ func (c *CommanderSingle) extract_tgz(qp string, sessionID int) (string, error)
 		return "", err
 	}
 
-	session_query_pack_tgz_filepath, err := c.st.Storage.SaveQueryPack(tgz, sessionID)
+	session_query_pack_tgz_filepath, err := c.st.ServerStore.SaveQueryPack(tgz, sessionID)
 	if err != nil {
 		return "", err
 	}

pkg/server/types.go

@@ -1,9 +1,9 @@
 package server
 
 import (
-	"mrvacommander/pkg/agent"
 	"mrvacommander/pkg/common"
 	"mrvacommander/pkg/logger"
+	"mrvacommander/pkg/qpstore"
 	"mrvacommander/pkg/queue"
 	"mrvacommander/pkg/storage"
 )
@@ -26,19 +26,28 @@ type SessionInfo struct {
 }
 
 type CommanderSingle struct {
-	st    *State
+	st *Visibles
-	queue queue.Queue
 }
 
-func NewCommanderSingle(s *State, q queue.Queue) *CommanderSingle {
+func NewCommanderSingle() *CommanderSingle {
-	c := CommanderSingle{s, q}
+	c := CommanderSingle{}
 	return &c
 }
 
-type State struct {
+// type State struct {
-	Commander Commander
+// 	Commander Commander
-	Logger    logger.Logger
+// 	Logger    logger.Logger
-	Queue     queue.Queue
+// 	Queue     queue.Queue
-	Storage   storage.Storage
+// 	Storage   storage.Storage
-	Runner    agent.Runner
+// 	Runner    agent.Runner
+// }
+
+type Visibles struct {
+	Logger      logger.Logger
+	Queue       queue.Queue
+	ServerStore storage.Storage
+	// TODO extra package for query pack storage
+	QueryPackStore qpstore.Storage
+	// TODO extra package for ql db storage
+	QLDBStore storage.Storage
 }

pkg/storage/container.go

@@ -32,6 +32,15 @@ func (s *StorageContainer) FindAvailableDBs(analysisReposRequested []common.Owne
 	return notFoundRepos, analysisRepos
 }
 
+func (s *StorageContainer) Setup(v *Visibles) {
+	s.modules = v
+}
+
+func NewQLDBStore() (*StorageContainer, error) {
+	// TODO set up qldb_db
+	return nil, nil
+}
+
 func NewStorageContainer(startingID int) (*StorageContainer, error) {
 
 	db, err := ConnectDB(DBSpec{
@@ -39,32 +48,21 @@ func NewStorageContainer(startingID int) (*StorageContainer, error) {
 		Port:     5432,
 		User:     "exampleuser",
 		Password: "examplepass",
-		DBname:   "exampledb",
+		DBname:   "server_db",
 	})
-
 	if err != nil {
 		return nil, err
 	}
 
-	s, err := LoadOrInit(db, startingID)
-	if err != nil {
-		return nil, err
-	}
-	return s, nil
-
-}
-
-func LoadOrInit(db *gorm.DB, startingID int) (*StorageContainer, error) {
-	// Check and set up the database
 	s := StorageContainer{RequestID: startingID, DB: db}
-	if s.hasTables() {
+	if err := s.SetupDB(); err != nil {
-		s.loadState()
+		return nil, err
-	} else {
-		if err := s.SetupDB(); err != nil {
-			return nil, err
-		}
-		s.setFresh()
 	}
+
+	if err = s.loadState(); err != nil {
+		return nil, err
+	}
+
 	return &s, nil
 }
 
@@ -80,12 +78,7 @@ func ConnectDB(s DBSpec) (*gorm.DB, error) {
 	return db, nil
 }
 
-func (s *StorageContainer) setFresh() {
-	// TODO Set initial state
-}
-
 func (s *StorageContainer) SetupDB() error {
-	// TODO Migrate the schemas
 	msg := "Failed to initialize database "
 
 	if err := s.DB.AutoMigrate(&DBInfo{}); err != nil {
@@ -108,9 +101,9 @@ func (s *StorageContainer) SetupDB() error {
 	return nil
 }
 
-func (s *StorageContainer) loadState() {
+func (s *StorageContainer) loadState() error {
 	// TODO load the state
-	return
+	return nil
 }
 
 func (s *StorageContainer) hasTables() bool {

pkg/storage/storage.go

@@ -28,6 +28,10 @@ func NewStorageSingle(startingID int) *StorageSingle {
 	return &s
 }
 
+func (s *StorageSingle) Setup(v *Visibles) {
+	s.modules = v
+}
+
 func (s *StorageSingle) NextID() int {
 	s.currentID += 1
 	return s.currentID
@@ -111,6 +115,9 @@ func ArtifactURL(js common.JobSpec, vaid int) (string, error) {
 		slog.Error("Error packaging results:", "error", err)
 		return "", err
 	}
+	// TODO Need url valid in container network and externally
+	// For now, we assume the container port 8080 is port 8080 on user's machine
+	hostname = "localhost"
 	au := fmt.Sprintf("http://%s:8080/download-server/%s", hostname, zfpath)
 	return au, nil
 }

pkg/storage/types.go

@@ -13,6 +13,7 @@ type DBLocation struct {
 
 type StorageSingle struct {
 	currentID int
+	modules   *Visibles
 }
 
 type DBSpec struct {
@@ -59,4 +60,7 @@ type StorageContainer struct {
 	// Database version of StorageSingle
 	RequestID int
 	DB        *gorm.DB
+	modules   *Visibles
 }
+
+type Visibles struct{}

postgres-init-scripts/dbinit.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    SELECT 'CREATE DATABASE server_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'server_db')\gexec
+    SELECT 'CREATE DATABASE querypack_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'querypack_db')\gexec
+    SELECT 'CREATE DATABASE qldb_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'qldb_db')\gexec
+EOSQL