Compare commits

..

10 Commits

Author SHA1 Message Date
8f5976286a wip: update state_postgres, update dependent modules 2025-07-11 15:34:07 -07:00
d2a716d4a8 update main.go: remove 'standalone' mode 2025-07-11 13:34:36 -07:00
Michael Hohn
cdde32cc8f switch to pollling rabbitmq; update images 2025-06-16 16:07:37 -07:00
Michael Hohn
f2d0c9fd82 minor workspace config change 2025-05-06 18:12:24 -07:00
Michael Hohn
1b3ca2e9da Add option MRVA_HEPC_DATAVIACLI, get hepc data using cli tool 2025-04-09 19:20:10 -07:00
Michael Hohn
0b82dab461 added to .gitignore 2025-02-10 11:31:28 -08:00
Michael Hohn
fb18d7fff8 added .gitignore 2025-02-10 11:30:47 -08:00
Michael Hohn
0235e47241 Add hepc integration 2025-01-30 14:40:46 -08:00
Michael Hohn
1a8f34f893 updates 2025-01-22 10:39:25 -08:00
Michael Hohn
2963095939 Use brewer color scheme 2025-01-22 10:27:36 -08:00
9 changed files with 89 additions and 82 deletions

4
.gitignore vendored Normal file
View File

@@ -0,0 +1,4 @@
.DS_Store
mrvaserver
*~

View File

@@ -1,10 +1,11 @@
all: mrvaserver
.PHONY: all clean msla
all: msla
msla:
GOOS=linux GOARCH=arm64 go build
mrvaserver:
# GOOS=linux GOARCH=arm64 go build
go build
clean:

View File

@@ -1,9 +1,22 @@
// dot -Tpdf Makefile.dot > Makefile.pdf && open Makefile.pdf
digraph {
"all" [shape=parallelogram style=solid color="#e41a1c" penwidth=2] // Red
"msla" [shape=parallelogram style=solid color="#377eb8" penwidth=2] // Blue
"mrvaserver" [shape=rectangle style=solid color="#4daf4a" penwidth=2] // Green
"clean" [shape=parallelogram style=solid color="#984ea3" penwidth=2] // Purple
node [colorscheme=set14]
// Regular targets
"all" [shape=parallelogram style=solid color=1 penwidth=2]
"msla" [shape=parallelogram style=solid color=2 penwidth=2]
"mrvaserver" [shape=rectangle style=solid color=3 penwidth=2]
"clean" [shape=parallelogram style=solid color=4 penwidth=2]
// Phony nodes
"all_phony" [shape=ellipse style=solid color=black]
"msla_phony" [shape=ellipse style=solid color=black]
"clean_phony" [shape=ellipse style=solid color=black]
// Dependencies
"all" -> "msla"
"all" -> "all_phony"
"msla" -> "msla_phony"
"clean" -> "clean_phony"
}

BIN
Makefile.pdf Normal file

Binary file not shown.

22
go.mod
View File

@@ -2,32 +2,30 @@ module mrvaserver
go 1.22.0
require (
github.com/elastic/go-sysinfo v1.14.0
github.com/google/uuid v1.6.0
github.com/gorilla/mux v1.8.1
github.com/hohn/mrvacommander v0.2.1
)
require github.com/hohn/mrvacommander v0.2.1
require (
github.com/BurntSushi/toml v1.4.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/elastic/go-windows v1.0.1 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/mux v1.8.1 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/pgx/v5 v5.6.0 // indirect
github.com/jackc/puddle/v2 v2.2.1 // indirect
github.com/klauspost/compress v1.17.6 // indirect
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/minio-go/v7 v7.0.71 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/rabbitmq/amqp091-go v1.10.0 // indirect
github.com/rs/xid v1.5.0 // indirect
golang.org/x/crypto v0.24.0 // indirect
golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect
golang.org/x/net v0.23.0 // indirect
golang.org/x/sync v0.9.0 // indirect
golang.org/x/sys v0.21.0 // indirect
golang.org/x/text v0.16.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
howett.net/plist v1.0.1 // indirect
)
replace github.com/hohn/mrvacommander => /Users/hohn/work-gh/mrva/mrvacommander

35
go.sum
View File

@@ -1,25 +1,22 @@
github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/elastic/go-sysinfo v1.14.0 h1:dQRtiqLycoOOla7IflZg3aN213vqJmP0lpVpKQ9lUEY=
github.com/elastic/go-sysinfo v1.14.0/go.mod h1:FKUXnZWhnYI0ueO7jhsGV3uQJ5hiz8OqM5b3oGyaRr8=
github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0=
github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss=
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
github.com/hohn/mrvacommander v0.2.1 h1:HsP2Cq+pJ2e2FFagbfP9z60gvGwd1aG1srPMQ41A/Z4=
github.com/hohn/mrvacommander v0.2.1/go.mod h1:q3rqsmLGhtjrTinUEXS1z2mwMPaTH99WeTQ45zyhdBQ=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY=
github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw=
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
@@ -29,21 +26,17 @@ github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.71 h1:No9XfOKTYi6i0GnBj+WZwD8WP5GZfL7n7GOjRqCdAjA=
github.com/minio/minio-go/v7 v7.0.71/go.mod h1:4yBA8v80xGA30cfM3fz0DKYMXunWl/AV/6tWEs9ryzo=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
@@ -52,18 +45,16 @@ golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+h
golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak=
golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM=
howett.net/plist v1.0.1/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=

53
main.go
View File

@@ -4,23 +4,15 @@
package main
import (
"context"
"flag"
"log"
"log/slog"
"os"
"os/signal"
"path/filepath"
"sync"
"syscall"
"github.com/hohn/mrvacommander/config/mcc"
"github.com/hohn/mrvacommander/pkg/agent"
"github.com/hohn/mrvacommander/pkg/artifactstore"
"github.com/hohn/mrvacommander/pkg/deploy"
"github.com/hohn/mrvacommander/pkg/qldbstore"
"github.com/hohn/mrvacommander/pkg/queue"
"github.com/hohn/mrvacommander/pkg/server"
"github.com/hohn/mrvacommander/pkg/state"
)
@@ -28,8 +20,8 @@ import (
func main() {
// Define flags
helpFlag := flag.Bool("help", false, "Display help message")
logLevel := flag.String("loglevel", "info", "Set log level: debug, info, warn, error")
mode := flag.String("mode", "standalone", "Set mode: standalone, container, cluster")
logLevel := flag.String("loglevel", "debug", "Set log level: debug, info, warn, error")
mode := flag.String("mode", "container", "Set mode: standalone, container, cluster")
dbPathRoot := flag.String("dbpath", "", "Set the root path for the database store if using standalone mode.")
// Custom usage function for the help flag
@@ -78,8 +70,8 @@ func main() {
slog.Info("Using default database root path", "dbPathRoot", *dbPathRoot)
}
// Read configuration
config := mcc.LoadConfig("mcconfig.toml")
// // Read configuration
// config := mcc.LoadConfig("mcconfig.toml")
// Output configuration summary
log.Printf("Help: %t\n", *helpFlag)
@@ -93,30 +85,8 @@ func main() {
// Apply 'mode' flag
switch *mode {
case "standalone":
// Assemble single-process version
sq := queue.NewQueueSingle(2)
ss := state.NewLocalState(config.Storage.StartingID)
as := artifactstore.NewInMemoryArtifactStore()
ql := qldbstore.NewLocalFilesystemCodeQLDatabaseStore(*dbPathRoot)
server.NewCommanderSingle(&server.Visibles{
Queue: sq,
State: ss,
Artifacts: as,
CodeQLDBStore: ql,
})
var wg sync.WaitGroup
ctx, cancel := context.WithCancel(context.Background())
go agent.StartAndMonitorWorkers(ctx, as, ql, sq, 2, &wg)
slog.Info("Started server and standalone agent")
<-sigChan
slog.Info("Shutting down...")
cancel()
wg.Wait()
slog.Info("Agent shutdown complete")
slog.Error("--mode standalone is deprecated. Allowed values are: container, cluster")
os.Exit(1)
case "container":
isAgent := false
@@ -134,15 +104,22 @@ func main() {
os.Exit(1)
}
databases, err := deploy.InitMinIOCodeQLDatabaseStore()
databases, err := deploy.InitHEPCDatabaseStore()
if err != nil {
slog.Error("Failed to initialize database store", slog.Any("error", err))
os.Exit(1)
}
// server.NewCommanderSingle(&server.Visibles{
// Queue: rabbitMQQueue,
// State: state.NewLocalState(config.Storage.StartingID),
// Artifacts: artifacts,
// CodeQLDBStore: databases,
// })
server.NewCommanderSingle(&server.Visibles{
Queue: rabbitMQQueue,
State: state.NewLocalState(config.Storage.StartingID),
State: state.NewPGState(),
Artifacts: artifacts,
CodeQLDBStore: databases,
})

View File

@@ -1,10 +1,17 @@
{
"folders": [
{
"path": "."
}
{
"path": "."
},
{
"path": "../mrvacommander"
},
{
"path": "../mrvaagent"
}
],
"settings": {
"makefile.configureOnOpen": false
"makefile.configureOnOpen": false,
"sarif-viewer.connectToGithubCodeScanning": "off"
}
}

16
run-spigot.sh Executable file
View File

@@ -0,0 +1,16 @@
#!/bin/sh
# Usage: ./run-spigot.sh <refroot-dir> <out-dir> <tool-name>
# To be used with ./spigot-cli.go
set -eu
REFROOT_DIR="$1"
OUT_DIR="$2"
TOOL_NAME="$3"
cd "$REFROOT_DIR" && \
dpkg-refroot-env \
spigot-cli bulk-download-results \
--tool-name "$TOOL_NAME" \
--metadata-only all \
> "$OUT_DIR/spigot-results.json"