diff --git a/README.md b/README.md index ecbdb04..8ebceef 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,9 @@ TODO Style notes - NO package init() functions - Dynamic behaviour must be explicit -## cross-compile server on host, run it in container +## Cross-compile server on host, run it in container +These are simple steps using a single container. + 1. build server on host GOOS=linux GOARCH=arm64 go build @@ -27,21 +29,42 @@ TODO Style notes cd /mrva/mrvacommander/cmd/server/ && ./server -## Using docker -1. start the services +## Using docker-compose +Steps to build and run the server in a multi-container environment set up by docker-compose. +1. Build server on host + + cd ~/work-gh/mrva/mrvacommander/cmd/server/ + GOOS=linux GOARCH=arm64 go build + +1. Start the containers + + cd ~/work-gh/mrva/mrvacommander/ docker-compose up -d - -2. get status +4. Run server in its container + + cd ~/work-gh/mrva/mrvacommander/ + docker exec -it server bash + cd /mrva/mrvacommander/cmd/server/ + ./server -loglevel=debug -mode=container + +1. Test server via remote client by following the steps in [gh-mrva](https://github.com/hohn/gh-mrva/blob/connection-redirect/README.org#compacted-edit-run-debug-cycle) + + + + +Some general docker-compose commands + +2. Get service status docker-compose ps -3. stop services +3. Stop services docker-compose down -4. view all logs +4. View all logs docker-compose logs @@ -50,8 +73,11 @@ TODO Style notes docker exec -it server bash curl -I postgres:5432 curl -I http://rabbitmq:15672 - -1. Accessing PostgreSQL + + +Some postgres specific commands + +1. Access PostgreSQL psql -h localhost -p 5432 -U exampleuser -d exampledb @@ -59,6 +85,8 @@ TODO Style notes \dt +To run pgmin, the minimal go/postgres test part of this repository: + 1. Run pgmin ```sh diff --git a/cmd/server/main.go b/cmd/server/main.go index 80f6eef..b3a9720 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -85,7 +85,28 @@ func main() { sc.Setup(&state) // sc is part of state and dereferences it case "container": - // Assemble cccontainer + // Assemble container version + sq := queue.NewQueueSingle(2) // FIXME take value from configuration + sc := server.NewCommanderSingle(nil, sq) + sl := logger.NewLoggerSingle() + ss, err := storage.NewStorageContainer(config.Storage.StartingID) + if err != nil { + slog.Error("Unable to initialize storage") + os.Exit(1) + } + + sr := agent.NewRunnerSingle(2, sq) // FIXME take value from configuration + + state := server.State{ + Commander: sc, + Logger: sl, + Queue: sq, + Storage: ss, + Runner: sr, + } + + sc.Setup(&state) // sc is part of state and dereferences it + case "cluster": // Assemble cccluster default: diff --git a/pkg/server/server.go b/pkg/server/server.go index 816a657..f7b88c6 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -54,6 +54,7 @@ func (c *CommanderSingle) Setup(st *State) { // // Bind to a port and pass our router in // + // TODO make this a configuration entry log.Fatal(http.ListenAndServe(":8080", r)) } diff --git a/pkg/storage/container.go b/pkg/storage/container.go new file mode 100644 index 0000000..a693c65 --- /dev/null +++ b/pkg/storage/container.go @@ -0,0 +1,314 @@ +package storage + +import ( + "fmt" + "log/slog" + "mrvacommander/pkg/common" + "sync" + + "gorm.io/driver/postgres" + "gorm.io/gorm" +) + +var ( + DBmutex sync.Mutex +) + +func (s *StorageContainer) NextID() int { + // TODO update via db + return 12345 +} + +func (s *StorageContainer) SaveQueryPack(tgz []byte, sessionID int) (storagePath string, error error) { + // TODO save and return path + return "todo:no-path-yet", nil +} + +func (s *StorageContainer) FindAvailableDBs(analysisReposRequested []common.OwnerRepo) (notFoundRepos []common.OwnerRepo, analysisRepos *map[common.OwnerRepo]DBLocation) { + // TODO s.FindAvailableDBs() via postgres + analysisRepos = &map[common.OwnerRepo]DBLocation{} + notFoundRepos = []common.OwnerRepo{} + + return notFoundRepos, analysisRepos +} + +func NewStorageContainer(startingID int) (*StorageContainer, error) { + // Set up the database connection string + const ( + host = "postgres" + port = 5432 + user = "exampleuser" + password = "examplepass" + dbname = "exampledb" + ) + + // Open the database connection + dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", + host, port, user, password, dbname) + db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{}) + if err != nil { + slog.Error("Error connecting to the database", "err", err) + return nil, err + } + + // Check and set up the database + s := StorageContainer{RequestID: startingID, DB: db} + if s.hasTables() { + s.loadState() + } else { + if err = s.setupDB(); err != nil { + return nil, err + } + s.setFresh() + } + + return &s, nil +} + +func (s *StorageContainer) setFresh() { + // TODO Set initial state +} + +func (s *StorageContainer) setupDB() error { + // TODO Migrate the schemas + msg := "Failed to initialize database " + + if err := s.DB.AutoMigrate(&DBInfo{}); err != nil { + slog.Error(msg, "table", "dbinfo") + return err + } + if err := s.DB.AutoMigrate(&DBJobs{}); err != nil { + slog.Error(msg, "table", "dbjobs") + return err + } + if err := s.DB.AutoMigrate(&DBResult{}); err != nil { + slog.Error(msg, "table", "dbresult") + return err + } + if err := s.DB.AutoMigrate(&DBStatus{}); err != nil { + slog.Error(msg, "table", "dbstatus") + return err + } + + return nil +} + +func (s *StorageContainer) loadState() { + // TODO load the state + return +} + +func (s *StorageContainer) hasTables() bool { + // TODO sql query to check for tables + return false +} + +// ================ TODO migrate + +// func (s *StorageSingle) NextID() int { +// s.RequestID += 1 +// return s.RequestID +// } + +// func (s *StorageSingle) SaveQueryPack(tgz []byte, sessionId int) (string, error) { +// // Save the tar.gz body +// cwd, err := os.Getwd() +// if err != nil { +// slog.Error("No working directory") +// panic(err) +// } + +// dirpath := path.Join(cwd, "var", "codeql", "querypacks") +// if err := os.MkdirAll(dirpath, 0755); err != nil { +// slog.Error("Unable to create query pack output directory", +// "dir", dirpath) +// return "", err +// } + +// fpath := path.Join(dirpath, fmt.Sprintf("qp-%d.tgz", sessionId)) +// err = os.WriteFile(fpath, tgz, 0644) +// if err != nil { +// slog.Error("unable to save querypack body decoding error", "path", fpath) +// return "", err +// } else { +// slog.Info("Query pack saved to ", "path", fpath) +// } + +// return fpath, nil +// } + +// // Determine for which repositories codeql databases are available. +// // +// // Those will be the analysis_repos. The rest will be skipped. +// func (s *StorageSingle) FindAvailableDBs(analysisReposRequested []common.OwnerRepo) (notFoundRepos []common.OwnerRepo, +// analysisRepos *map[common.OwnerRepo]DBLocation) { +// slog.Debug("Looking for available CodeQL databases") + +// cwd, err := os.Getwd() +// if err != nil { +// slog.Error("No working directory") +// return +// } + +// analysisRepos = &map[common.OwnerRepo]DBLocation{} + +// notFoundRepos = []common.OwnerRepo{} + +// for _, rep := range analysisReposRequested { +// dbPrefix := filepath.Join(cwd, "codeql", "dbs", rep.Owner, rep.Repo) +// dbName := fmt.Sprintf("%s_%s_db.zip", rep.Owner, rep.Repo) +// dbPath := filepath.Join(dbPrefix, dbName) + +// if _, err := os.Stat(dbPath); errors.Is(err, fs.ErrNotExist) { +// slog.Info("Database does not exist for repository ", "owner/repo", rep, +// "path", dbPath) +// notFoundRepos = append(notFoundRepos, rep) +// } else { +// slog.Info("Found database for ", "owner/repo", rep, "path", dbPath) +// (*analysisRepos)[rep] = DBLocation{Prefix: dbPrefix, File: dbName} +// } +// } +// return notFoundRepos, analysisRepos +// } + +// func ArtifactURL(js common.JobSpec, vaid int) (string, error) { +// // We're looking for paths like +// // codeql/sarif/google/flatbuffers/google_flatbuffers.sarif + +// ar := GetResult(js) + +// hostname, err := os.Hostname() +// if err != nil { +// slog.Error("No host name found") +// return "", nil +// } + +// zfpath, err := PackageResults(ar, js.OwnerRepo, vaid) +// if err != nil { +// slog.Error("Error packaging results:", "error", err) +// return "", err +// } +// au := fmt.Sprintf("http://%s:8080/download-server/%s", hostname, zfpath) +// return au, nil +// } + +// func GetResult(js common.JobSpec) common.AnalyzeResult { +// mutex.Lock() +// defer mutex.Unlock() +// ar := result[js] +// return ar +// } + +// func SetResult(sessionid int, orl common.OwnerRepo, ar common.AnalyzeResult) { +// mutex.Lock() +// defer mutex.Unlock() +// result[common.JobSpec{RequestID: sessionid, OwnerRepo: orl}] = ar +// } + +// func PackageResults(ar common.AnalyzeResult, owre common.OwnerRepo, vaid int) (zipPath string, e error) { +// slog.Debug("Readying zip file with .sarif/.bqrs", "analyze-result", ar) + +// cwd, err := os.Getwd() +// if err != nil { +// slog.Error("No working directory") +// panic(err) +// } + +// // Ensure the output directory exists +// dirpath := path.Join(cwd, "var", "codeql", "localrun", "results") +// if err := os.MkdirAll(dirpath, 0755); err != nil { +// slog.Error("Unable to create results output directory", +// "dir", dirpath) +// return "", err +// } + +// // Create a new zip file +// zpath := path.Join(dirpath, fmt.Sprintf("results-%s-%s-%d.zip", owre.Owner, owre.Repo, vaid)) + +// zfile, err := os.Create(zpath) +// if err != nil { +// return "", err +// } +// defer zfile.Close() + +// // Create a new zip writer +// zwriter := zip.NewWriter(zfile) +// defer zwriter.Close() + +// // Add each result file to the zip archive +// names := []([]string){{ar.RunAnalysisSARIF, "results.sarif"}} +// for _, fpath := range names { +// file, err := os.Open(fpath[0]) +// if err != nil { +// return "", err +// } +// defer file.Close() + +// // Create a new file in the zip archive with custom name +// // The client is very specific: +// // if zf.Name != "results.sarif" && zf.Name != "results.bqrs" { continue } + +// zipEntry, err := zwriter.Create(fpath[1]) +// if err != nil { +// return "", err +// } + +// // Copy the contents of the file to the zip entry +// _, err = io.Copy(zipEntry, file) +// if err != nil { +// return "", err +// } +// } +// return zpath, nil +// } + +// func GetJobList(sessionid int) []common.AnalyzeJob { +// mutex.Lock() +// defer mutex.Unlock() +// return jobs[sessionid] +// } + +// func GetJobInfo(js common.JobSpec) common.JobInfo { +// mutex.Lock() +// defer mutex.Unlock() +// return info[js] +// } + +// func SetJobInfo(js common.JobSpec, ji common.JobInfo) { +// mutex.Lock() +// defer mutex.Unlock() +// info[js] = ji +// } + +// func GetStatus(sessionid int, orl common.OwnerRepo) common.Status { +// mutex.Lock() +// defer mutex.Unlock() +// return status[common.JobSpec{RequestID: sessionid, OwnerRepo: orl}] +// } + +// func ResultAsFile(path string) (string, []byte, error) { +// fpath := path +// if !filepath.IsAbs(path) { +// fpath = "/" + path +// } + +// file, err := os.ReadFile(fpath) +// if err != nil { +// slog.Warn("Failed to read results file", fpath, err) +// return "", nil, err +// } + +// return fpath, file, nil +// } + +// func SetStatus(sessionid int, orl common.OwnerRepo, s common.Status) { +// mutex.Lock() +// defer mutex.Unlock() +// status[common.JobSpec{RequestID: sessionid, OwnerRepo: orl}] = s +// } + +// func AddJob(sessionid int, job common.AnalyzeJob) { +// mutex.Lock() +// defer mutex.Unlock() +// jobs[sessionid] = append(jobs[sessionid], job) +// } diff --git a/pkg/storage/interfaces.go b/pkg/storage/interfaces.go index 82be054..5f16b62 100644 --- a/pkg/storage/interfaces.go +++ b/pkg/storage/interfaces.go @@ -1 +1,10 @@ package storage + +import "mrvacommander/pkg/common" + +type Storage interface { + NextID() int + SaveQueryPack(tgz []byte, sessionID int) (storagePath string, error error) + FindAvailableDBs(analysisReposRequested []common.OwnerRepo) (not_found_repos []common.OwnerRepo, + analysisRepos *map[common.OwnerRepo]DBLocation) +} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index d2cc2f4..1bd5575 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -23,18 +23,14 @@ var ( mutex sync.Mutex ) -type StorageSingle struct { - CurrentID int -} - func NewStorageSingle(startingID int) *StorageSingle { - s := StorageSingle{CurrentID: startingID} + s := StorageSingle{currentID: startingID} return &s } func (s *StorageSingle) NextID() int { - s.CurrentID += 1 - return s.CurrentID + s.currentID += 1 + return s.currentID } func (s *StorageSingle) SaveQueryPack(tgz []byte, sessionId int) (string, error) { diff --git a/pkg/storage/types.go b/pkg/storage/types.go index 1305ce1..b1c5509 100644 --- a/pkg/storage/types.go +++ b/pkg/storage/types.go @@ -2,16 +2,53 @@ package storage import ( "mrvacommander/pkg/common" -) -type Storage interface { - NextID() int - SaveQueryPack(tgz []byte, sessionID int) (storagePath string, error error) - FindAvailableDBs(analysisReposRequested []common.OwnerRepo) (not_found_repos []common.OwnerRepo, - analysisRepos *map[common.OwnerRepo]DBLocation) -} + "gorm.io/gorm" +) type DBLocation struct { Prefix string File string } + +type StorageSingle struct { + currentID int +} + +type DBInfo struct { + // Database version of + // info map[common.JobSpec]common.JobInfo = make(map[common.JobSpec]common.JobInfo) + gorm.Model + Key common.JobSpec + JobInfo common.JobInfo +} + +type DBJobs struct { + // Database version of + // jobs map[int][]common.AnalyzeJob = make(map[int][]common.AnalyzeJob) + gorm.Model + Key int + AnalyzeJob common.AnalyzeJob +} + +type DBResult struct { + // Database version of + // result map[common.JobSpec]common.AnalyzeResult = make(map[common.JobSpec]common.AnalyzeResult) + gorm.Model + Key common.JobSpec + AnalyzeResult common.AnalyzeResult +} + +type DBStatus struct { + // Database version of + // status map[common.JobSpec]common.Status = make(map[common.JobSpec]common.Status) + gorm.Model + Key common.JobSpec + Status common.Status +} + +type StorageContainer struct { + // Database version of StorageSingle + RequestID int + DB *gorm.DB +}