Fully implement local and container MRVA
This commit is contained in:
Nicolas Will
@@ -1,38 +1,56 @@
|
||||
# Use the ubuntu 22.04 base image
|
||||
FROM ubuntu:24.10
|
||||
FROM golang:1.22 AS builder
|
||||
|
||||
# Set architecture to arm64
|
||||
ARG ARCH=arm64
|
||||
ARG AARCH=aarch64
|
||||
# Copy the entire project
|
||||
WORKDIR /app
|
||||
COPY . .
|
||||
|
||||
# Set environment variables
|
||||
# Download dependencies
|
||||
RUN go mod download
|
||||
|
||||
# Set the working directory to the cmd/server subproject
|
||||
WORKDIR /app/cmd/server
|
||||
|
||||
# Build the server
|
||||
RUN go build -o /bin/mrva_server ./main.go
|
||||
|
||||
FROM ubuntu:24.10 as runner
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
ENV CODEQL_VERSION=codeql-bundle-v2.17.5
|
||||
ENV CODEQL_DOWNLOAD_URL=https://github.com/github/codeql-action/releases/download/${CODEQL_VERSION}/codeql-bundle-linux64.tar.gz
|
||||
ENV JDK_VERSION=22.0.1
|
||||
ENV JDK_DOWNLOAD_URL=https://download.oracle.com/java/21/latest/jdk-${JDK_VERSION}_linux-${AARCH}_bin.tar.gz
|
||||
ENV JDK_DOWNLOAD_URL=https://download.java.net/java/GA/jdk${JDK_VERSION}/c7ec1332f7bb44aeba2eb341ae18aca4/8/GPL/openjdk-${JDK_VERSION}_linux-${AARCH}_bin.tar.gz
|
||||
|
||||
ENV CODEQL_JAVA_HOME=/usr/local/jdk-${JDK_VERSION}
|
||||
# Build argument for CodeQL version, defaulting to the latest release
|
||||
ARG CODEQL_VERSION=latest
|
||||
|
||||
# Install necessary tools
|
||||
RUN apt-get update && \
|
||||
apt-get install -y curl tar && \
|
||||
apt-get clean && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
# Install packages
|
||||
RUN apt-get update && apt-get install --no-install-recommends --assume-yes \
|
||||
unzip \
|
||||
curl \
|
||||
ca-certificates \
|
||||
default-jdk
|
||||
|
||||
# Add and extract the CodeQL bundle
|
||||
RUN curl -L $CODEQL_DOWNLOAD_URL -o /tmp/${CODEQL_VERSION}.tar.gz && \
|
||||
tar -xzf /tmp/${CODEQL_VERSION}.tar.gz -C /opt && \
|
||||
rm /tmp/${CODEQL_VERSION}.tar.gz
|
||||
# If the version is 'latest', lsget the latest release version from GitHub, unzip the bundle into /opt, and delete the archive
|
||||
RUN if [ "$CODEQL_VERSION" = "latest" ]; then \
|
||||
CODEQL_VERSION=$(curl -s https://api.github.com/repos/github/codeql-cli-binaries/releases/latest | grep '"tag_name"' | sed -E 's/.*"([^"]+)".*/\1/'); \
|
||||
fi && \
|
||||
echo "Using CodeQL version $CODEQL_VERSION" && \
|
||||
curl -L "https://github.com/github/codeql-cli-binaries/releases/download/$CODEQL_VERSION/codeql-linux64.zip" -o /tmp/codeql.zip && \
|
||||
unzip /tmp/codeql.zip -d /opt && \
|
||||
rm /tmp/codeql.zip && \
|
||||
chmod -R +x /opt/codeql
|
||||
|
||||
# Add and extract the JDK
|
||||
RUN curl -L $JDK_DOWNLOAD_URL -o /tmp/jdk-${JDK_VERSION}.tar.gz && \
|
||||
tar -xzf /tmp/jdk-${JDK_VERSION}.tar.gz -C /usr/local && \
|
||||
rm /tmp/jdk-${JDK_VERSION}.tar.gz
|
||||
# Set environment variables for CodeQL
|
||||
ENV CODEQL_CLI_PATH=/opt/codeql/codeql
|
||||
|
||||
# Set PATH
|
||||
ENV PATH=/opt/codeql:"$PATH"
|
||||
# Set environment variable for CodeQL for `codeql database analyze` support on ARM
|
||||
# This env var has no functional effect on CodeQL when running on x86_64 linux
|
||||
ENV CODEQL_JAVA_HOME=/usr
|
||||
|
||||
# Prepare host mount point
|
||||
RUN mkdir /mrva
|
||||
# Set working directory to /app
|
||||
|
||||
# Copy built server binary from the builder stage
|
||||
COPY --from=builder /bin/mrva_server ./mrva_server
|
||||
|
||||
# Copy the CodeQL database directory from the builder stage (for standalone mode)
|
||||
COPY --from=builder /app/cmd/server/codeql ./codeql
|
||||
|
||||
# Run the server with the default mode set to container
|
||||
ENTRYPOINT ["./mrva_server"]
|
||||
CMD ["--mode=container"]
|
||||
@@ -4,20 +4,25 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"log"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"syscall"
|
||||
|
||||
"mrvacommander/config/mcc"
|
||||
|
||||
"mrvacommander/pkg/agent"
|
||||
"mrvacommander/pkg/logger"
|
||||
"mrvacommander/pkg/artifactstore"
|
||||
"mrvacommander/pkg/deploy"
|
||||
"mrvacommander/pkg/qldbstore"
|
||||
"mrvacommander/pkg/qpstore"
|
||||
"mrvacommander/pkg/queue"
|
||||
"mrvacommander/pkg/server"
|
||||
"mrvacommander/pkg/storage"
|
||||
"mrvacommander/pkg/state"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -25,13 +30,14 @@ func main() {
|
||||
helpFlag := flag.Bool("help", false, "Display help message")
|
||||
logLevel := flag.String("loglevel", "info", "Set log level: debug, info, warn, error")
|
||||
mode := flag.String("mode", "standalone", "Set mode: standalone, container, cluster")
|
||||
dbPathRoot := flag.String("dbpath", "", "Set the root path for the database store if using standalone mode.")
|
||||
|
||||
// Custom usage function for the help flag
|
||||
flag.Usage = func() {
|
||||
log.Printf("Usage of %s:\n", os.Args[0])
|
||||
flag.PrintDefaults()
|
||||
log.Println("\nExamples:")
|
||||
log.Println(" go run main.go --loglevel=debug --mode=container")
|
||||
log.Println("go run main.go --loglevel=debug --mode=container --dbpath=/path/to/db_dir")
|
||||
}
|
||||
|
||||
// Parse the flags
|
||||
@@ -58,6 +64,20 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Process database root if standalone and not provided
|
||||
if *mode == "standalone" && *dbPathRoot == "" {
|
||||
slog.Warn("No database root path provided.")
|
||||
// Current directory of the Executable has a codeql directory. There.
|
||||
// Resolve the absolute directory based on os.Executable()
|
||||
execPath, err := os.Executable()
|
||||
if err != nil {
|
||||
slog.Error("Failed to get executable path", slog.Any("error", err))
|
||||
os.Exit(1)
|
||||
}
|
||||
*dbPathRoot = filepath.Dir(execPath) + "/codeql/dbs/"
|
||||
slog.Info("Using default database root path", "dbPathRoot", *dbPathRoot)
|
||||
}
|
||||
|
||||
// Read configuration
|
||||
config := mcc.LoadConfig("mcconfig.toml")
|
||||
|
||||
@@ -66,91 +86,73 @@ func main() {
|
||||
log.Printf("Log Level: %s\n", *logLevel)
|
||||
log.Printf("Mode: %s\n", *mode)
|
||||
|
||||
// Handle signals
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
// Apply 'mode' flag
|
||||
switch *mode {
|
||||
case "standalone":
|
||||
// Assemble single-process version
|
||||
|
||||
sl := logger.NewLoggerSingle(&logger.Visibles{})
|
||||
|
||||
// FIXME take value from configuration
|
||||
sq := queue.NewQueueSingle(2, &queue.Visibles{
|
||||
Logger: sl,
|
||||
})
|
||||
|
||||
ss := storage.NewStorageSingle(config.Storage.StartingID, &storage.Visibles{})
|
||||
|
||||
qp, err := qpstore.NewStore(&qpstore.Visibles{})
|
||||
if err != nil {
|
||||
slog.Error("Unable to initialize query pack storage")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
ql, err := qldbstore.NewStore(&qldbstore.Visibles{})
|
||||
if err != nil {
|
||||
slog.Error("Unable to initialize ql database storage")
|
||||
os.Exit(1)
|
||||
}
|
||||
sq := queue.NewQueueSingle(2)
|
||||
ss := state.NewLocalState(config.Storage.StartingID)
|
||||
as := artifactstore.NewInMemoryArtifactStore()
|
||||
ql := qldbstore.NewLocalFilesystemCodeQLDatabaseStore(*dbPathRoot)
|
||||
|
||||
server.NewCommanderSingle(&server.Visibles{
|
||||
Logger: sl,
|
||||
Queue: sq,
|
||||
ServerStore: ss,
|
||||
QueryPackStore: qp,
|
||||
QLDBStore: ql,
|
||||
Queue: sq,
|
||||
State: ss,
|
||||
Artifacts: as,
|
||||
CodeQLDBStore: ql,
|
||||
})
|
||||
|
||||
// FIXME take value from configuration
|
||||
agent.NewAgentSingle(2, &agent.Visibles{
|
||||
Logger: sl,
|
||||
Queue: sq,
|
||||
QueryPackStore: qp,
|
||||
QLDBStore: ql,
|
||||
})
|
||||
var wg sync.WaitGroup
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
go agent.StartAndMonitorWorkers(ctx, as, ql, sq, 2, &wg)
|
||||
|
||||
slog.Info("Started server and standalone agent")
|
||||
<-sigChan
|
||||
slog.Info("Shutting down...")
|
||||
cancel()
|
||||
wg.Wait()
|
||||
slog.Info("Agent shutdown complete")
|
||||
|
||||
case "container":
|
||||
// Assemble container version
|
||||
sl := logger.NewLoggerSingle(&logger.Visibles{})
|
||||
isAgent := false
|
||||
|
||||
// FIXME take value from configuration
|
||||
sq := queue.NewQueueSingle(2, &queue.Visibles{
|
||||
Logger: sl,
|
||||
})
|
||||
|
||||
ss := storage.NewStorageSingle(config.Storage.StartingID, &storage.Visibles{})
|
||||
|
||||
qp, err := qpstore.NewStore(&qpstore.Visibles{})
|
||||
rabbitMQQueue, err := deploy.InitRabbitMQ(isAgent)
|
||||
if err != nil {
|
||||
slog.Error("Unable to initialize query pack storage")
|
||||
slog.Error("Failed to initialize RabbitMQ", slog.Any("error", err))
|
||||
os.Exit(1)
|
||||
}
|
||||
defer rabbitMQQueue.Close()
|
||||
|
||||
artifacts, err := deploy.InitMinIOArtifactStore()
|
||||
if err != nil {
|
||||
slog.Error("Failed to initialize artifact store", slog.Any("error", err))
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
ql, err := qldbstore.NewStore(&qldbstore.Visibles{})
|
||||
databases, err := deploy.InitMinIOCodeQLDatabaseStore()
|
||||
if err != nil {
|
||||
slog.Error("Unable to initialize ql database storage")
|
||||
slog.Error("Failed to initialize database store", slog.Any("error", err))
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
agent.NewAgentSingle(2, &agent.Visibles{
|
||||
Logger: sl,
|
||||
Queue: sq,
|
||||
QueryPackStore: qp,
|
||||
QLDBStore: ql,
|
||||
})
|
||||
|
||||
server.NewCommanderSingle(&server.Visibles{
|
||||
Logger: sl,
|
||||
Queue: sq,
|
||||
ServerStore: ss,
|
||||
QueryPackStore: qp,
|
||||
QLDBStore: ql,
|
||||
Queue: rabbitMQQueue,
|
||||
State: state.NewLocalState(config.Storage.StartingID),
|
||||
Artifacts: artifacts,
|
||||
CodeQLDBStore: databases,
|
||||
})
|
||||
|
||||
case "cluster":
|
||||
// Assemble cluster version
|
||||
slog.Info("Started server in container mode.")
|
||||
<-sigChan
|
||||
default:
|
||||
slog.Error("Invalid value for --mode. Allowed values are: standalone, container, cluster\n")
|
||||
slog.Error("Invalid value for --mode. Allowed values are: standalone, container, cluster")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
slog.Info("Server shutdown complete")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user