mrva/mrvacommander
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #12 from advanced-security/hohn-0.1.11

Browse Source 
Hohn 0.1.11  Make cross-module visibility explicit
This commit is contained in:
Michael Hohn
2024-06-12 11:41:17 -07:00
committed by GitHub
parent 25cab583c1 5730c330f4
commit 5b9df73424
16 changed files with 443 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
README.md
View File

@@ -15,6 +15,7 @@ These are simple steps using a single container.
2. build docker image
cd cmd/server
docker build -t server-image .
3. Start container with shared directory
@@ -30,7 +31,9 @@ These are simple steps using a single container.
cd /mrva/mrvacommander/cmd/server/ && ./server
## Using docker-compose
Steps to build and run the server in a multi-container environment set up by docker-compose.
### Steps to build and run the server in a multi-container environment set up by docker-compose.
1. Built the server-image, above
1. Build server on host
@@ -52,10 +55,7 @@ Steps to build and run the server in a multi-container environment set up by doc
1. Test server via remote client by following the steps in [gh-mrva](https://github.com/hohn/gh-mrva/blob/connection-redirect/README.org#compacted-edit-run-debug-cycle)
Some general docker-compose commands
### Some general docker-compose commands
2. Get service status
@@ -76,7 +76,7 @@ Some general docker-compose commands
curl -I http://rabbitmq:15672
Some postgres specific commands
### Some postgres specific commands
1. Access PostgreSQL
@@ -86,7 +86,49 @@ Some postgres specific commands
\dt
To run pgmin, the minimal go/postgres test part of this repository:
1. Examine a table
select * from db_infos
1. Show all columns in a specific table
\d+ db_infos
1. Miscellany
\pset pager off
\lo_import FILE [COMMENT]
### Manually create needed postgres databases
docker-compose now runs a db init script, but this information is useful for
debugging/manual work.
~~This is still necessary after `docker-compose up` to avoid~~
[error] failed to initialize database, got error failed to connect to
`user=exampleuser database=server_db`: 172.25.0.3:5432 (postgres): server
error: FATAL: database "server_db" does not exist (SQLSTATE 3D000)
from
./server -loglevel=debug -mode=container
The steps:
# on the host
psql -h localhost -p 5432 -U exampleuser -d postgres
# Conditionally create dbs
SELECT 'CREATE DATABASE server_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'server_db')\gexec
SELECT 'CREATE DATABASE querypack_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'querypack_db')\gexec
SELECT 'CREATE DATABASE qldb_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'qldb_db')\gexec
# List all dbs
\l
### To run pgmin, the minimal go/postgres test part of this repository
1. Run pgmin

40
cmd/server/Dockerfile
View File

@@ -1,14 +1,38 @@
# Use Ubuntu 22.04 as the base image
FROM arm64v8/ubuntu:22.04
# Use the ubuntu 22.04 base image
FROM ubuntu:22.04
# Set environment variables to non-interactive to avoid prompts during installation
# Set architecture to arm64
ARG ARCH=arm64
ARG AARCH=aarch64
# Set environment variables
ENV DEBIAN_FRONTEND=noninteractive
ENV CODEQL_VERSION=codeql-bundle-v2.17.5
ENV CODEQL_DOWNLOAD_URL=https://github.com/github/codeql-action/releases/download/${CODEQL_VERSION}/codeql-bundle-linux64.tar.gz
ENV JDK_VERSION=22.0.1
ENV JDK_DOWNLOAD_URL=https://download.oracle.com/java/21/latest/jdk-${JDK_VERSION}_linux-${AARCH}_bin.tar.gz
ENV JDK_DOWNLOAD_URL=https://download.java.net/java/GA/jdk${JDK_VERSION}/c7ec1332f7bb44aeba2eb341ae18aca4/8/GPL/openjdk-${JDK_VERSION}_linux-${AARCH}_bin.tar.gz
# Update the package list
ENV CODEQL_JAVA_HOME=/usr/local/jdk-${JDK_VERSION}
# Install necessary tools
RUN apt-get update && \
apt-get clean
apt-get install -y curl tar && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
# Add and extract the CodeQL bundle
RUN curl -L $CODEQL_DOWNLOAD_URL -o /tmp/${CODEQL_VERSION}.tar.gz && \
tar -xzf /tmp/${CODEQL_VERSION}.tar.gz -C /opt && \
rm /tmp/${CODEQL_VERSION}.tar.gz
# Add and extract the JDK
RUN curl -L $JDK_DOWNLOAD_URL -o /tmp/jdk-${JDK_VERSION}.tar.gz && \
tar -xzf /tmp/jdk-${JDK_VERSION}.tar.gz -C /usr/local && \
rm /tmp/jdk-${JDK_VERSION}.tar.gz
# Set PATH
ENV PATH=/opt/codeql:"$PATH"
# Prepare host mount point
RUN mkdir /mrva
# Set the default command
CMD ["bash"]

84
cmd/server/main.go
View File

@@ -13,6 +13,7 @@ import (
"mrvacommander/pkg/agent"
"mrvacommander/pkg/logger"
"mrvacommander/pkg/qpstore"
"mrvacommander/pkg/queue"
"mrvacommander/pkg/server"
"mrvacommander/pkg/storage"
@@ -69,43 +70,94 @@ func main() {
case "standalone":
// Assemble single-process version
sq := queue.NewQueueSingle(2) // FIXME take value from configuration
sc := server.NewCommanderSingle(nil, sq)
sc := server.NewCommanderSingle()
sl := logger.NewLoggerSingle()
ss := storage.NewStorageSingle(config.Storage.StartingID)
sr := agent.NewRunnerSingle(2, sq) // FIXME take value from configuration
state := server.State{
Commander: sc,
Logger: sl,
Queue: sq,
Storage: ss,
Runner: sr,
qp, err := qpstore.NewStore(config.Storage.StartingID)
if err != nil {
slog.Error("Unable to initialize query pack storage")
os.Exit(1)
}
sc.Setup(&state) // sc is part of state and dereferences it
ql, err := storage.NewQLDBStore()
if err != nil {
slog.Error("Unable to initialize ql database storage")
os.Exit(1)
}
sc.Setup(&server.Visibles{
Logger: sl,
Queue: sq,
ServerStore: ss,
QueryPackStore: qp,
QLDBStore: ql,
})
sl.Setup(&logger.Visibles{})
sq.Setup(&queue.Visibles{
Logger: sl,
})
ss.Setup(&storage.Visibles{})
sr.Setup(&agent.Visibles{
Logger: sl,
Queue: sq,
QueryPackStore: qp,
QLDBStore: ql,
})
case "container":
// Assemble container version
sq := queue.NewQueueSingle(2) // FIXME take value from configuration
sc := server.NewCommanderSingle(nil, sq)
sc := server.NewCommanderSingle()
sl := logger.NewLoggerSingle()
ss, err := storage.NewStorageContainer(config.Storage.StartingID)
if err != nil {
slog.Error("Unable to initialize storage")
slog.Error("Unable to initialize server storage")
os.Exit(1)
}
qp, err := qpstore.NewStore(config.Storage.StartingID)
if err != nil {
slog.Error("Unable to initialize query pack storage")
os.Exit(1)
}
ql, err := storage.NewQLDBStore()
if err != nil {
slog.Error("Unable to initialize ql database storage")
os.Exit(1)
}
sr := agent.NewRunnerSingle(2, sq) // FIXME take value from configuration
state := server.State{
Commander: sc,
sc.Setup(&server.Visibles{
Logger: sl,
Queue: sq,
Storage: ss,
Runner: sr,
}
ServerStore: ss,
QueryPackStore: qp,
QLDBStore: ql,
})
sc.Setup(&state) // sc is part of state and dereferences it
sl.Setup(&logger.Visibles{})
sq.Setup(&queue.Visibles{
Logger: sl,
})
ss.Setup(&storage.Visibles{})
sr.Setup(&agent.Visibles{
Logger: sl,
Queue: sq,
QueryPackStore: qp,
QLDBStore: ql,
})
case "cluster":
// Assemble cccluster

35
docker-compose.yml
View File

@@ -9,7 +9,8 @@ services:
POSTGRES_PASSWORD: examplepass
POSTGRES_DB: exampledb
volumes:
- postgres_data:/Users/hohn/var/lib/postgresql/data
- postgres_data:/var/lib/postgresql/data
- ./postgres-init-scripts:/docker-entrypoint-initdb.d
ports:
- "5432:5432" # Exposing PostgreSQL to the host
expose:
@@ -34,11 +35,13 @@ services:
server:
image: ubuntu:22.04
image: server-image
container_name: server
command: sh -c "apt-get update && apt-get install -y curl && tail -f /dev/null"
environment:
- MRVA_SERVER_ROOT=/mrva/mrvacommander/cmd/server
command: sh -c "tail -f /dev/null"
ports:
- "8080:80" # Exposing port 80 inside the container as port 8080 on the host
- "8080:8080"
volumes:
- /Users/hohn/work-gh/mrva/mrvacommander:/mrva/mrvacommander
depends_on:
@@ -47,10 +50,28 @@ services:
networks:
- backend
minio:
image: minio/minio:RELEASE.2024-06-11T03-13-30Z
container_name: minio
ports:
- "9000:9000"
- "9001:9001"
environment:
MINIO_ROOT_USER: user
MINIO_ROOT_PASSWORD: musty-coke
command: server /data --console-address ":9001"
volumes:
- minio-data:/data
volumes:
minio-data:
postgres_data:
driver: local
networks:
backend:
driver: bridge
volumes:
postgres_data:
driver: local

34
pkg/agent/agent.go
View File

@@ -2,6 +2,8 @@ package agent
import (
"mrvacommander/pkg/common"
"mrvacommander/pkg/logger"
"mrvacommander/pkg/qpstore"
"mrvacommander/pkg/queue"
"mrvacommander/pkg/storage"
@@ -31,6 +33,19 @@ func NewRunnerSingle(numWorkers int, queue queue.Queue) *RunnerSingle {
return &r
}
type Visibles struct {
Logger logger.Logger
Queue queue.Queue
// TODO extra package for query pack storage
QueryPackStore qpstore.Storage
// TODO extra package for ql db storage
QLDBStore storage.Storage
}
func (c *RunnerSingle) Setup(st *Visibles) {
return
}
func (r *RunnerSingle) worker(wid int) {
var job common.AnalyzeJob
@@ -67,21 +82,20 @@ func (r *RunnerSingle) RunAnalysis(job common.AnalyzeJob) (string, error) {
queryPackID, dbOwner, dbRepo :=
job.QueryPackId, job.ORepo.Owner, job.ORepo.Repo
// FIXME Provide this via environment or explicit argument
gmsRoot := "/Users/hohn/work-gh/mrva/mrvacommander/cmd/server"
serverRoot := os.Getenv("MRVA_SERVER_ROOT")
// Set up derived paths
dbPath := filepath.Join(gmsRoot, "var/codeql/dbs", dbOwner, dbRepo)
dbZip := filepath.Join(gmsRoot, "codeql/dbs", dbOwner, dbRepo,
dbPath := filepath.Join(serverRoot, "var/codeql/dbs", dbOwner, dbRepo)
dbZip := filepath.Join(serverRoot, "codeql/dbs", dbOwner, dbRepo,
fmt.Sprintf("%s_%s_db.zip", dbOwner, dbRepo))
dbExtract := filepath.Join(gmsRoot, "var/codeql/dbs", dbOwner, dbRepo)
dbExtract := filepath.Join(serverRoot, "var/codeql/dbs", dbOwner, dbRepo)
queryPack := filepath.Join(gmsRoot,
queryPack := filepath.Join(serverRoot,
"var/codeql/querypacks", fmt.Sprintf("qp-%d.tgz", queryPackID))
queryExtract := filepath.Join(gmsRoot,
queryExtract := filepath.Join(serverRoot,
"var/codeql/querypacks", fmt.Sprintf("qp-%d", queryPackID))
queryOutDir := filepath.Join(gmsRoot,
queryOutDir := filepath.Join(serverRoot,
"var/codeql/sarif/localrun", dbOwner, dbRepo)
queryOutFile := filepath.Join(queryOutDir,
fmt.Sprintf("%s_%s.sarif", dbOwner, dbRepo))
@@ -93,7 +107,7 @@ func (r *RunnerSingle) RunAnalysis(job common.AnalyzeJob) (string, error) {
}
if err := unzipFile(dbZip, dbExtract); err != nil {
slog.Error("Failed to unzip DB %s: %v", dbZip, err)
slog.Error("Failed to unzip DB", dbZip, err)
return "", err
}
@@ -118,7 +132,7 @@ func (r *RunnerSingle) RunAnalysis(job common.AnalyzeJob) (string, error) {
cmd := exec.Command("codeql", "database", "analyze",
"--format=sarif-latest", "--rerun", "--output", queryOutFile,
"-j8", dbPath, queryExtract)
cmd.Dir = gmsRoot
cmd.Dir = serverRoot
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr

7
pkg/logger/types.go
View File

@@ -1,9 +1,16 @@
package logger
type LoggerSingle struct {
modules *Visibles
}
func NewLoggerSingle() *LoggerSingle {
l := LoggerSingle{}
return &l
}
type Visibles struct{}
func (l *LoggerSingle) Setup(v *Visibles) {
l.modules = v
}

6
pkg/qldbstore/interfaces.go Normal file
View File

@@ -0,0 +1,6 @@
package qldbstore
type DBLocation struct {
Prefix string
File string
}

138
pkg/qpstore/container.go Normal file
View File

@@ -0,0 +1,138 @@
package qpstore
import (
"fmt"
"log/slog"
"mrvacommander/pkg/common"
"mrvacommander/pkg/qldbstore"
"sync"
"gorm.io/driver/postgres"
"gorm.io/gorm"
)
var (
DBmutex sync.Mutex
)
type StorageVisibles struct{}
type StorageContainer struct {
// Database version of StorageSingle
RequestID int
DB *gorm.DB
modules *StorageVisibles
}
type DBSpec struct {
Host string
Port int
User string
Password string
DBname string
}
func (s *StorageContainer) SetupDB() error {
// TODO set up query pack storage
return nil
}
func (s *StorageContainer) LoadState() error {
// TODO load the state
return nil
}
func (s *StorageContainer) hasTables() bool {
// TODO query to check for tables
return false
}
func (s *StorageContainer) NextID() int {
// TODO update via db
return 12345
}
func (s *StorageContainer) SaveQueryPack(tgz []byte, sessionID int) (storagePath string, error error) {
// TODO save and return path
return "todo:no-path-yet", nil
}
func (s *StorageContainer) FindAvailableDBs(analysisReposRequested []common.OwnerRepo) (notFoundRepos []common.OwnerRepo, analysisRepos *map[common.OwnerRepo]qldbstore.DBLocation) {
// TODO s.FindAvailableDBs() via postgres
analysisRepos = &map[common.OwnerRepo]qldbstore.DBLocation{}
notFoundRepos = []common.OwnerRepo{}
return notFoundRepos, analysisRepos
}
func (s *StorageContainer) Setup(v *StorageVisibles) {
s.modules = v
}
func NewStore(startingID int) (Storage, error) {
// TODO drop the startingID
db, err := ConnectDB(DBSpec{
Host: "postgres",
Port: 5432,
User: "exampleuser",
Password: "examplepass",
DBname: "querypack_db",
})
if err != nil {
return nil, err
}
s := StorageContainer{RequestID: startingID, DB: db}
if err := s.SetupDB(); err != nil {
return nil, err
}
if err = s.loadState(); err != nil {
return nil, err
}
return &s, nil
}
func NewStorageContainer(startingID int) (*StorageContainer, error) {
db, err := ConnectDB(DBSpec{
Host: "postgres",
Port: 5432,
User: "exampleuser",
Password: "examplepass",
DBname: "server_db",
})
if err != nil {
return nil, err
}
s := StorageContainer{RequestID: startingID, DB: db}
if err := s.SetupDB(); err != nil {
return nil, err
}
if err = s.loadState(); err != nil {
return nil, err
}
return &s, nil
}
func ConnectDB(s DBSpec) (*gorm.DB, error) {
// Open the database connection
dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
s.Host, s.Port, s.User, s.Password, s.DBname)
db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{})
if err != nil {
slog.Error("Error connecting to the database", "err", err)
return nil, err
}
return db, nil
}
func (s *StorageContainer) loadState() error {
// TODO load the state
return nil
}

13
pkg/qpstore/interfaces.go Normal file
View File

@@ -0,0 +1,13 @@
package qpstore
import (
"mrvacommander/pkg/common"
"mrvacommander/pkg/qldbstore"
)
type Storage interface {
NextID() int
SaveQueryPack(tgz []byte, sessionID int) (storagePath string, error error)
FindAvailableDBs(analysisReposRequested []common.OwnerRepo) (not_found_repos []common.OwnerRepo,
analysisRepos *map[common.OwnerRepo]qldbstore.DBLocation)
}

14
pkg/queue/types.go
View File

@@ -1,11 +1,23 @@
package queue
import "mrvacommander/pkg/common"
import (
"mrvacommander/pkg/common"
"mrvacommander/pkg/logger"
)
type QueueSingle struct {
NumWorkers int
jobs chan common.AnalyzeJob
results chan common.AnalyzeResult
modules *Visibles
}
type Visibles struct {
Logger logger.Logger
}
func (q *QueueSingle) Setup(v *Visibles) {
q.modules = v
}
func NewQueueSingle(numWorkers int) *QueueSingle {

10
pkg/server/server.go
View File

@@ -21,7 +21,7 @@ import (
"github.com/gorilla/mux"
)
func (c *CommanderSingle) Setup(st *State) {
func (c *CommanderSingle) Setup(st *Visibles) {
r := mux.NewRouter()
c.st = st
@@ -275,7 +275,7 @@ func (c *CommanderSingle) MirvaRequest(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
slog.Info("New mrva run ", "owner", vars["owner"], "repo", vars["repo"])
session_id := c.st.Storage.NextID()
session_id := c.st.ServerStore.NextID()
session_owner := vars["owner"]
session_controller_repo := vars["repo"]
slog.Info("new run", "id: ", fmt.Sprint(session_id), session_owner, session_controller_repo)
@@ -284,9 +284,9 @@ func (c *CommanderSingle) MirvaRequest(w http.ResponseWriter, r *http.Request) {
return
}
not_found_repos, analysisRepos := c.st.Storage.FindAvailableDBs(session_repositories)
not_found_repos, analysisRepos := c.st.ServerStore.FindAvailableDBs(session_repositories)
c.queue.StartAnalyses(analysisRepos, session_id, session_language)
c.st.Queue.StartAnalyses(analysisRepos, session_id, session_language)
si := SessionInfo{
ID: session_id,
@@ -492,7 +492,7 @@ func (c *CommanderSingle) extract_tgz(qp string, sessionID int) (string, error)
return "", err
}
session_query_pack_tgz_filepath, err := c.st.Storage.SaveQueryPack(tgz, sessionID)
session_query_pack_tgz_filepath, err := c.st.ServerStore.SaveQueryPack(tgz, sessionID)
if err != nil {
return "", err
}

27
pkg/server/types.go
View File

@@ -1,9 +1,9 @@
package server
import (
"mrvacommander/pkg/agent"
"mrvacommander/pkg/common"
"mrvacommander/pkg/logger"
"mrvacommander/pkg/qpstore"
"mrvacommander/pkg/queue"
"mrvacommander/pkg/storage"
)
@@ -26,19 +26,28 @@ type SessionInfo struct {
}
type CommanderSingle struct {
st *State
queue queue.Queue
st *Visibles
}
func NewCommanderSingle(s *State, q queue.Queue) *CommanderSingle {
c := CommanderSingle{s, q}
func NewCommanderSingle() *CommanderSingle {
c := CommanderSingle{}
return &c
}
type State struct {
Commander Commander
// type State struct {
// Commander Commander
// Logger logger.Logger
// Queue queue.Queue
// Storage storage.Storage
// Runner agent.Runner
// }
type Visibles struct {
Logger logger.Logger
Queue queue.Queue
Storage storage.Storage
Runner agent.Runner
ServerStore storage.Storage
// TODO extra package for query pack storage
QueryPackStore qpstore.Storage
// TODO extra package for ql db storage
QLDBStore storage.Storage
}

39
pkg/storage/container.go
View File

@@ -32,6 +32,15 @@ func (s *StorageContainer) FindAvailableDBs(analysisReposRequested []common.Owne
return notFoundRepos, analysisRepos
}
func (s *StorageContainer) Setup(v *Visibles) {
s.modules = v
}
func NewQLDBStore() (*StorageContainer, error) {
// TODO set up qldb_db
return nil, nil
}
func NewStorageContainer(startingID int) (*StorageContainer, error) {
db, err := ConnectDB(DBSpec{
@@ -39,32 +48,21 @@ func NewStorageContainer(startingID int) (*StorageContainer, error) {
Port: 5432,
User: "exampleuser",
Password: "examplepass",
DBname: "exampledb",
DBname: "server_db",
})
if err != nil {
return nil, err
}
s, err := LoadOrInit(db, startingID)
if err != nil {
return nil, err
}
return s, nil
}
func LoadOrInit(db *gorm.DB, startingID int) (*StorageContainer, error) {
// Check and set up the database
s := StorageContainer{RequestID: startingID, DB: db}
if s.hasTables() {
s.loadState()
} else {
if err := s.SetupDB(); err != nil {
return nil, err
}
s.setFresh()
if err = s.loadState(); err != nil {
return nil, err
}
return &s, nil
}
@@ -80,12 +78,7 @@ func ConnectDB(s DBSpec) (*gorm.DB, error) {
return db, nil
}
func (s *StorageContainer) setFresh() {
// TODO Set initial state
}
func (s *StorageContainer) SetupDB() error {
// TODO Migrate the schemas
msg := "Failed to initialize database "
if err := s.DB.AutoMigrate(&DBInfo{}); err != nil {
@@ -108,9 +101,9 @@ func (s *StorageContainer) SetupDB() error {
return nil
}
func (s *StorageContainer) loadState() {
func (s *StorageContainer) loadState() error {
// TODO load the state
return
return nil
}
func (s *StorageContainer) hasTables() bool {

7
pkg/storage/storage.go
View File

@@ -28,6 +28,10 @@ func NewStorageSingle(startingID int) *StorageSingle {
return &s
}
func (s *StorageSingle) Setup(v *Visibles) {
s.modules = v
}
func (s *StorageSingle) NextID() int {
s.currentID += 1
return s.currentID
@@ -111,6 +115,9 @@ func ArtifactURL(js common.JobSpec, vaid int) (string, error) {
slog.Error("Error packaging results:", "error", err)
return "", err
}
// TODO Need url valid in container network and externally
// For now, we assume the container port 8080 is port 8080 on user's machine
hostname = "localhost"
au := fmt.Sprintf("http://%s:8080/download-server/%s", hostname, zfpath)
return au, nil
}

4
pkg/storage/types.go
View File

@@ -13,6 +13,7 @@ type DBLocation struct {
type StorageSingle struct {
currentID int
modules *Visibles
}
type DBSpec struct {
@@ -59,4 +60,7 @@ type StorageContainer struct {
// Database version of StorageSingle
RequestID int
DB *gorm.DB
modules *Visibles
}
type Visibles struct{}

8
postgres-init-scripts/dbinit.sh Executable file
View File

@@ -0,0 +1,8 @@
#!/bin/bash
set -e
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
SELECT 'CREATE DATABASE server_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'server_db')\gexec
SELECT 'CREATE DATABASE querypack_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'querypack_db')\gexec
SELECT 'CREATE DATABASE qldb_db' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'qldb_db')\gexec
EOSQL