mrva/mrvacommander
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fully implement local and container MRVA

Browse Source
This commit is contained in:
Nicolas Will
2024-06-17 13:16:24 +02:00
parent ef7552c43f
commit e0cbc01d21
43 changed files with 1700 additions and 1137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
cmd/agent/Dockerfile
View File

@@ -23,7 +23,8 @@ ARG CODEQL_VERSION=latest
RUN apt-get update && apt-get install --no-install-recommends --assume-yes \
unzip \
curl \
ca-certificates
ca-certificates \
default-jdk
# If the version is 'latest', lsget the latest release version from GitHub, unzip the bundle into /opt, and delete the archive
RUN if [ "$CODEQL_VERSION" = "latest" ]; then \
@@ -32,14 +33,15 @@ RUN if [ "$CODEQL_VERSION" = "latest" ]; then \
echo "Using CodeQL version $CODEQL_VERSION" && \
curl -L "https://github.com/github/codeql-cli-binaries/releases/download/$CODEQL_VERSION/codeql-linux64.zip" -o /tmp/codeql.zip && \
unzip /tmp/codeql.zip -d /opt && \
rm /tmp/codeql.zip
rm /tmp/codeql.zip && \
chmod -R +x /opt/codeql
# Set environment variables for CodeQL
ENV CODEQL_CLI_PATH=/opt/codeql
ENV CODEQL_CLI_PATH=/opt/codeql/codeql
# Set environment variable for CodeQL for `codeql database analyze` support on ARM
# This env var has no functional effect on CodeQL when running on x86_64 linux
ENV CODEQL_JAVA_HOME=/usr/
ENV CODEQL_JAVA_HOME=/usr
# Copy built agent binary from the builder stage
WORKDIR /app

165
cmd/agent/main.go
View File

@@ -3,170 +3,71 @@ package main
import (
"context"
"flag"
"log"
"log/slog"
"os"
"os/signal"
"runtime"
"strconv"
"sync"
"syscall"
"time"
"github.com/elastic/go-sysinfo"
"golang.org/x/exp/slog"
"mrvacommander/pkg/agent"
"mrvacommander/pkg/queue"
"mrvacommander/pkg/deploy"
)
const (
workerMemoryMB = 2048 // 2 GB
monitorIntervalSec = 10 // Monitor every 10 seconds
)
func calculateWorkers() int {
host, err := sysinfo.Host()
if err != nil {
slog.Error("failed to get host info", "error", err)
os.Exit(1)
}
memInfo, err := host.Memory()
if err != nil {
slog.Error("failed to get memory info", "error", err)
os.Exit(1)
}
// Get available memory in MB
totalMemoryMB := memInfo.Available / (1024 * 1024)
// Ensure we have at least one worker
workers := int(totalMemoryMB / workerMemoryMB)
if workers < 1 {
workers = 1
}
// Limit the number of workers to the number of CPUs
cpuCount := runtime.NumCPU()
if workers > cpuCount {
workers = max(cpuCount, 1)
}
return workers
}
func startAndMonitorWorkers(ctx context.Context, queue queue.Queue, desiredWorkerCount int, wg *sync.WaitGroup) {
currentWorkerCount := 0
stopChans := make([]chan struct{}, 0)
if desiredWorkerCount != 0 {
slog.Info("Starting workers", slog.Int("count", desiredWorkerCount))
for i := 0; i < desiredWorkerCount; i++ {
stopChan := make(chan struct{})
stopChans = append(stopChans, stopChan)
wg.Add(1)
go agent.RunWorker(ctx, stopChan, queue, wg)
}
return
}
slog.Info("Worker count not specified, managing based on available memory and CPU")
for {
select {
case <-ctx.Done():
// signal all workers to stop
for _, stopChan := range stopChans {
close(stopChan)
}
return
default:
newWorkerCount := calculateWorkers()
if newWorkerCount != currentWorkerCount {
slog.Info(
"Modifying worker count",
slog.Int("current", currentWorkerCount),
slog.Int("new", newWorkerCount))
}
if newWorkerCount > currentWorkerCount {
for i := currentWorkerCount; i < newWorkerCount; i++ {
stopChan := make(chan struct{})
stopChans = append(stopChans, stopChan)
wg.Add(1)
go agent.RunWorker(ctx, stopChan, queue, wg)
}
} else if newWorkerCount < currentWorkerCount {
for i := newWorkerCount; i < currentWorkerCount; i++ {
close(stopChans[i])
}
stopChans = stopChans[:newWorkerCount]
}
currentWorkerCount = newWorkerCount
time.Sleep(monitorIntervalSec * time.Second)
}
}
}
func main() {
slog.Info("Starting agent")
workerCount := flag.Int("workers", 0, "number of workers")
logLevel := flag.String("loglevel", "info", "Set log level: debug, info, warn, error")
flag.Parse()
requiredEnvVars := []string{
"MRVA_RABBITMQ_HOST",
"MRVA_RABBITMQ_PORT",
"MRVA_RABBITMQ_USER",
"MRVA_RABBITMQ_PASSWORD",
"CODEQL_JAVA_HOME",
"CODEQL_CLI_PATH",
}
for _, envVar := range requiredEnvVars {
if _, ok := os.LookupEnv(envVar); !ok {
slog.Error("Missing required environment variable", "key", envVar)
os.Exit(1)
}
}
rmqHost := os.Getenv("MRVA_RABBITMQ_HOST")
rmqPort := os.Getenv("MRVA_RABBITMQ_PORT")
rmqUser := os.Getenv("MRVA_RABBITMQ_USER")
rmqPass := os.Getenv("MRVA_RABBITMQ_PASSWORD")
rmqPortAsInt, err := strconv.ParseInt(rmqPort, 10, 16)
if err != nil {
slog.Error("Failed to parse RabbitMQ port", slog.Any("error", err))
// Apply 'loglevel' flag
switch *logLevel {
case "debug":
slog.SetLogLoggerLevel(slog.LevelDebug)
case "info":
slog.SetLogLoggerLevel(slog.LevelInfo)
case "warn":
slog.SetLogLoggerLevel(slog.LevelWarn)
case "error":
slog.SetLogLoggerLevel(slog.LevelError)
default:
log.Printf("Invalid logging verbosity level: %s", *logLevel)
os.Exit(1)
}
slog.Info("Initializing RabbitMQ queue")
isAgent := true
rabbitMQQueue, err := queue.InitializeRabbitMQQueue(rmqHost, int16(rmqPortAsInt), rmqUser, rmqPass, false)
rabbitMQQueue, err := deploy.InitRabbitMQ(isAgent)
if err != nil {
slog.Error("failed to initialize RabbitMQ", slog.Any("error", err))
slog.Error("Failed to initialize RabbitMQ", slog.Any("error", err))
os.Exit(1)
}
defer rabbitMQQueue.Close()
artifacts, err := deploy.InitMinIOArtifactStore()
if err != nil {
slog.Error("Failed to initialize artifact store", slog.Any("error", err))
os.Exit(1)
}
databases, err := deploy.InitMinIOCodeQLDatabaseStore()
if err != nil {
slog.Error("Failed to initialize database store", slog.Any("error", err))
os.Exit(1)
}
var wg sync.WaitGroup
ctx, cancel := context.WithCancel(context.Background())
go startAndMonitorWorkers(ctx, rabbitMQQueue, *workerCount, &wg)
go agent.StartAndMonitorWorkers(ctx, artifacts, databases, rabbitMQQueue, *workerCount, &wg)
slog.Info("Agent started")
// Gracefully exit on SIGINT/SIGTERM
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
<-sigChan
slog.Info("Shutting down agent")
// TODO: fix this to gracefully terminate agent workers during jobs
slog.Info("Shutting down agent")
cancel()
wg.Wait()
slog.Info("Agent shutdown complete")
}

76
cmd/server/Dockerfile
View File

@@ -1,38 +1,56 @@
# Use the ubuntu 22.04 base image
FROM ubuntu:24.10
FROM golang:1.22 AS builder
# Set architecture to arm64
ARG ARCH=arm64
ARG AARCH=aarch64
# Copy the entire project
WORKDIR /app
COPY . .
# Set environment variables
# Download dependencies
RUN go mod download
# Set the working directory to the cmd/server subproject
WORKDIR /app/cmd/server
# Build the server
RUN go build -o /bin/mrva_server ./main.go
FROM ubuntu:24.10 as runner
ENV DEBIAN_FRONTEND=noninteractive
ENV CODEQL_VERSION=codeql-bundle-v2.17.5
ENV CODEQL_DOWNLOAD_URL=https://github.com/github/codeql-action/releases/download/${CODEQL_VERSION}/codeql-bundle-linux64.tar.gz
ENV JDK_VERSION=22.0.1
ENV JDK_DOWNLOAD_URL=https://download.oracle.com/java/21/latest/jdk-${JDK_VERSION}_linux-${AARCH}_bin.tar.gz
ENV JDK_DOWNLOAD_URL=https://download.java.net/java/GA/jdk${JDK_VERSION}/c7ec1332f7bb44aeba2eb341ae18aca4/8/GPL/openjdk-${JDK_VERSION}_linux-${AARCH}_bin.tar.gz
ENV CODEQL_JAVA_HOME=/usr/local/jdk-${JDK_VERSION}
# Build argument for CodeQL version, defaulting to the latest release
ARG CODEQL_VERSION=latest
# Install necessary tools
RUN apt-get update && \
apt-get install -y curl tar && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
# Install packages
RUN apt-get update && apt-get install --no-install-recommends --assume-yes \
unzip \
curl \
ca-certificates \
default-jdk
# Add and extract the CodeQL bundle
RUN curl -L $CODEQL_DOWNLOAD_URL -o /tmp/${CODEQL_VERSION}.tar.gz && \
tar -xzf /tmp/${CODEQL_VERSION}.tar.gz -C /opt && \
rm /tmp/${CODEQL_VERSION}.tar.gz
# If the version is 'latest', lsget the latest release version from GitHub, unzip the bundle into /opt, and delete the archive
RUN if [ "$CODEQL_VERSION" = "latest" ]; then \
CODEQL_VERSION=$(curl -s https://api.github.com/repos/github/codeql-cli-binaries/releases/latest | grep '"tag_name"' | sed -E 's/.*"([^"]+)".*/\1/'); \
fi && \
echo "Using CodeQL version $CODEQL_VERSION" && \
curl -L "https://github.com/github/codeql-cli-binaries/releases/download/$CODEQL_VERSION/codeql-linux64.zip" -o /tmp/codeql.zip && \
unzip /tmp/codeql.zip -d /opt && \
rm /tmp/codeql.zip && \
chmod -R +x /opt/codeql
# Add and extract the JDK
RUN curl -L $JDK_DOWNLOAD_URL -o /tmp/jdk-${JDK_VERSION}.tar.gz && \
tar -xzf /tmp/jdk-${JDK_VERSION}.tar.gz -C /usr/local && \
rm /tmp/jdk-${JDK_VERSION}.tar.gz
# Set environment variables for CodeQL
ENV CODEQL_CLI_PATH=/opt/codeql/codeql
# Set PATH
ENV PATH=/opt/codeql:"$PATH"
# Set environment variable for CodeQL for `codeql database analyze` support on ARM
# This env var has no functional effect on CodeQL when running on x86_64 linux
ENV CODEQL_JAVA_HOME=/usr
# Prepare host mount point
RUN mkdir /mrva
# Set working directory to /app
# Copy built server binary from the builder stage
COPY --from=builder /bin/mrva_server ./mrva_server
# Copy the CodeQL database directory from the builder stage (for standalone mode)
COPY --from=builder /app/cmd/server/codeql ./codeql
# Run the server with the default mode set to container
ENTRYPOINT ["./mrva_server"]
CMD ["--mode=container"]

132
cmd/server/main.go
View File

@@ -4,20 +4,25 @@
package main
import (
"context"
"flag"
"log"
"log/slog"
"os"
"os/signal"
"path/filepath"
"sync"
"syscall"
"mrvacommander/config/mcc"
"mrvacommander/pkg/agent"
"mrvacommander/pkg/logger"
"mrvacommander/pkg/artifactstore"
"mrvacommander/pkg/deploy"
"mrvacommander/pkg/qldbstore"
"mrvacommander/pkg/qpstore"
"mrvacommander/pkg/queue"
"mrvacommander/pkg/server"
"mrvacommander/pkg/storage"
"mrvacommander/pkg/state"
)
func main() {
@@ -25,13 +30,14 @@ func main() {
helpFlag := flag.Bool("help", false, "Display help message")
logLevel := flag.String("loglevel", "info", "Set log level: debug, info, warn, error")
mode := flag.String("mode", "standalone", "Set mode: standalone, container, cluster")
dbPathRoot := flag.String("dbpath", "", "Set the root path for the database store if using standalone mode.")
// Custom usage function for the help flag
flag.Usage = func() {
log.Printf("Usage of %s:\n", os.Args[0])
flag.PrintDefaults()
log.Println("\nExamples:")
log.Println(" go run main.go --loglevel=debug --mode=container")
log.Println("go run main.go --loglevel=debug --mode=container --dbpath=/path/to/db_dir")
}
// Parse the flags
@@ -58,6 +64,20 @@ func main() {
os.Exit(1)
}
// Process database root if standalone and not provided
if *mode == "standalone" && *dbPathRoot == "" {
slog.Warn("No database root path provided.")
// Current directory of the Executable has a codeql directory. There.
// Resolve the absolute directory based on os.Executable()
execPath, err := os.Executable()
if err != nil {
slog.Error("Failed to get executable path", slog.Any("error", err))
os.Exit(1)
}
*dbPathRoot = filepath.Dir(execPath) + "/codeql/dbs/"
slog.Info("Using default database root path", "dbPathRoot", *dbPathRoot)
}
// Read configuration
config := mcc.LoadConfig("mcconfig.toml")
@@ -66,91 +86,73 @@ func main() {
log.Printf("Log Level: %s\n", *logLevel)
log.Printf("Mode: %s\n", *mode)
// Handle signals
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
// Apply 'mode' flag
switch *mode {
case "standalone":
// Assemble single-process version
sl := logger.NewLoggerSingle(&logger.Visibles{})
// FIXME take value from configuration
sq := queue.NewQueueSingle(2, &queue.Visibles{
Logger: sl,
})
ss := storage.NewStorageSingle(config.Storage.StartingID, &storage.Visibles{})
qp, err := qpstore.NewStore(&qpstore.Visibles{})
if err != nil {
slog.Error("Unable to initialize query pack storage")
os.Exit(1)
}
ql, err := qldbstore.NewStore(&qldbstore.Visibles{})
if err != nil {
slog.Error("Unable to initialize ql database storage")
os.Exit(1)
}
sq := queue.NewQueueSingle(2)
ss := state.NewLocalState(config.Storage.StartingID)
as := artifactstore.NewInMemoryArtifactStore()
ql := qldbstore.NewLocalFilesystemCodeQLDatabaseStore(*dbPathRoot)
server.NewCommanderSingle(&server.Visibles{
Logger: sl,
Queue: sq,
ServerStore: ss,
QueryPackStore: qp,
QLDBStore: ql,
Queue: sq,
State: ss,
Artifacts: as,
CodeQLDBStore: ql,
})
// FIXME take value from configuration
agent.NewAgentSingle(2, &agent.Visibles{
Logger: sl,
Queue: sq,
QueryPackStore: qp,
QLDBStore: ql,
})
var wg sync.WaitGroup
ctx, cancel := context.WithCancel(context.Background())
go agent.StartAndMonitorWorkers(ctx, as, ql, sq, 2, &wg)
slog.Info("Started server and standalone agent")
<-sigChan
slog.Info("Shutting down...")
cancel()
wg.Wait()
slog.Info("Agent shutdown complete")
case "container":
// Assemble container version
sl := logger.NewLoggerSingle(&logger.Visibles{})
isAgent := false
// FIXME take value from configuration
sq := queue.NewQueueSingle(2, &queue.Visibles{
Logger: sl,
})
ss := storage.NewStorageSingle(config.Storage.StartingID, &storage.Visibles{})
qp, err := qpstore.NewStore(&qpstore.Visibles{})
rabbitMQQueue, err := deploy.InitRabbitMQ(isAgent)
if err != nil {
slog.Error("Unable to initialize query pack storage")
slog.Error("Failed to initialize RabbitMQ", slog.Any("error", err))
os.Exit(1)
}
defer rabbitMQQueue.Close()
artifacts, err := deploy.InitMinIOArtifactStore()
if err != nil {
slog.Error("Failed to initialize artifact store", slog.Any("error", err))
os.Exit(1)
}
ql, err := qldbstore.NewStore(&qldbstore.Visibles{})
databases, err := deploy.InitMinIOCodeQLDatabaseStore()
if err != nil {
slog.Error("Unable to initialize ql database storage")
slog.Error("Failed to initialize database store", slog.Any("error", err))
os.Exit(1)
}
agent.NewAgentSingle(2, &agent.Visibles{
Logger: sl,
Queue: sq,
QueryPackStore: qp,
QLDBStore: ql,
})
server.NewCommanderSingle(&server.Visibles{
Logger: sl,
Queue: sq,
ServerStore: ss,
QueryPackStore: qp,
QLDBStore: ql,
Queue: rabbitMQQueue,
State: state.NewLocalState(config.Storage.StartingID),
Artifacts: artifacts,
CodeQLDBStore: databases,
})
case "cluster":
// Assemble cluster version
slog.Info("Started server in container mode.")
<-sigChan
default:
slog.Error("Invalid value for --mode. Allowed values are: standalone, container, cluster\n")
slog.Error("Invalid value for --mode. Allowed values are: standalone, container, cluster")
os.Exit(1)
}
slog.Info("Server shutdown complete")
}