Fully implement local and container MRVA

pkg/artifactstore/common.go

@@ -0,0 +1,23 @@
+package artifactstore
+
+import (
+	"fmt"
+	"mrvacommander/pkg/common"
+)
+
+type ArtifactLocation struct {
+	// Data is a map of key-value pairs that describe the location of the artifact.
+	// For example, a simple key-value pair could be "path" -> "/path/to/artifact.tgz".
+	// Alternatively, a more complex example could be "bucket" -> "example", "key" -> "UNIQUE_ARTIFACT_IDENTIFIER".
+	Data map[string]string `json:"data"`
+}
+
+// deriveKeyFromSessionId generates a key for a query pack based on the job ID
+func deriveKeyFromSessionId(sessionId int) string {
+	return fmt.Sprintf("%d", sessionId)
+}
+
+// deriveKeyFromJobSpec generates a key for a result based on the JobSpec
+func deriveKeyFromJobSpec(jobSpec common.JobSpec) string {
+	return fmt.Sprintf("%d-%s", jobSpec.SessionID, jobSpec.NameWithOwner)
+}

pkg/artifactstore/interfaces.go

@@ -0,0 +1,20 @@
+package artifactstore
+
+import "mrvacommander/pkg/common"
+
+type Store interface {
+	// GetQueryPack retrieves the query pack from the specified location.
+	GetQueryPack(location ArtifactLocation) ([]byte, error)
+
+	// SaveQueryPack saves the query pack using the session ID and returns the artifact location.
+	SaveQueryPack(sessionId int, data []byte) (ArtifactLocation, error)
+
+	// GetResult retrieves the result from the specified location.
+	GetResult(location ArtifactLocation) ([]byte, error)
+
+	// GetResultSize retrieves the size of the result from the specified location.
+	GetResultSize(location ArtifactLocation) (int, error)
+
+	// SaveResult saves the result using the JobSpec and returns the artifact location.
+	SaveResult(jobSpec common.JobSpec, data []byte) (ArtifactLocation, error)
+}

pkg/artifactstore/store_memory.go

@@ -0,0 +1,94 @@
+package artifactstore
+
+import (
+	"fmt"
+	"mrvacommander/pkg/common"
+	"sync"
+)
+
+// InMemoryArtifactStore is an in-memory implementation of the ArtifactStore interface
+type InMemoryArtifactStore struct {
+	mu      sync.Mutex
+	packs   map[string][]byte
+	results map[string][]byte
+}
+
+func NewInMemoryArtifactStore() *InMemoryArtifactStore {
+	return &InMemoryArtifactStore{
+		packs:   make(map[string][]byte),
+		results: make(map[string][]byte),
+	}
+}
+
+// GetQueryPack retrieves the query pack from the specified location
+func (store *InMemoryArtifactStore) GetQueryPack(location ArtifactLocation) ([]byte, error) {
+	store.mu.Lock()
+	defer store.mu.Unlock()
+
+	key := location.Data["key"]
+	data, exists := store.packs[key]
+	if !exists {
+		return nil, fmt.Errorf("query pack not found: %s", key)
+	}
+	return data, nil
+}
+
+// SaveQueryPack saves the query pack using the session ID and returns the artifact location
+func (store *InMemoryArtifactStore) SaveQueryPack(sessionId int, data []byte) (ArtifactLocation, error) {
+	store.mu.Lock()
+	defer store.mu.Unlock()
+
+	key := deriveKeyFromSessionId(sessionId)
+	store.packs[key] = data
+
+	location := ArtifactLocation{
+		Data: map[string]string{
+			"bucket": "packs",
+			"key":    key,
+		},
+	}
+	return location, nil
+}
+
+// GetResult retrieves the result from the specified location
+func (store *InMemoryArtifactStore) GetResult(location ArtifactLocation) ([]byte, error) {
+	store.mu.Lock()
+	defer store.mu.Unlock()
+
+	key := location.Data["key"]
+	data, exists := store.results[key]
+	if !exists {
+		return nil, fmt.Errorf("result not found: %s", key)
+	}
+	return data, nil
+}
+
+// GetResultSize retrieves the size of the result from the specified location
+func (store *InMemoryArtifactStore) GetResultSize(location ArtifactLocation) (int, error) {
+	store.mu.Lock()
+	defer store.mu.Unlock()
+
+	key := location.Data["key"]
+	data, exists := store.results[key]
+	if !exists {
+		return 0, fmt.Errorf("result not found: %s", key)
+	}
+	return len(data), nil
+}
+
+// SaveResult saves the result using the JobSpec and returns the artifact location
+func (store *InMemoryArtifactStore) SaveResult(jobSpec common.JobSpec, data []byte) (ArtifactLocation, error) {
+	store.mu.Lock()
+	defer store.mu.Unlock()
+
+	key := deriveKeyFromJobSpec(jobSpec)
+	store.results[key] = data
+
+	location := ArtifactLocation{
+		Data: map[string]string{
+			"bucket": "results",
+			"key":    key,
+		},
+	}
+	return location, nil
+}

pkg/artifactstore/store_minio.go

@@ -0,0 +1,117 @@
+package artifactstore
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"log/slog"
+	"math"
+	"mrvacommander/pkg/common"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+const (
+	RESULTS_BUCKET_NAME = "results"
+	PACKS_BUCKET_NAME   = "packs"
+)
+
+type MinIOArtifactStore struct {
+	client *minio.Client
+}
+
+func NewMinIOArtifactStore(endpoint, id, secret string) (*MinIOArtifactStore, error) {
+	minioClient, err := minio.New(endpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(id, secret, ""),
+		Secure: false,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	slog.Info("Connected to MinIO artifact store server")
+
+	// Create "results" bucket
+	if err := common.CreateMinIOBucketIfNotExists(minioClient, RESULTS_BUCKET_NAME); err != nil {
+		return nil, fmt.Errorf("could not create results bucket: %v", err)
+	}
+
+	// Create "packs" bucket
+	if err := common.CreateMinIOBucketIfNotExists(minioClient, PACKS_BUCKET_NAME); err != nil {
+		return nil, fmt.Errorf("could not create packs bucket: %v", err)
+	}
+
+	return &MinIOArtifactStore{
+		client: minioClient,
+	}, nil
+}
+
+func (store *MinIOArtifactStore) GetQueryPack(location ArtifactLocation) ([]byte, error) {
+	return store.getArtifact(location)
+}
+
+func (store *MinIOArtifactStore) SaveQueryPack(jobId int, data []byte) (ArtifactLocation, error) {
+	return store.saveArtifact(PACKS_BUCKET_NAME, deriveKeyFromSessionId(jobId), data, "application/gzip")
+}
+
+func (store *MinIOArtifactStore) GetResult(location ArtifactLocation) ([]byte, error) {
+	return store.getArtifact(location)
+}
+
+func (store *MinIOArtifactStore) GetResultSize(location ArtifactLocation) (int, error) {
+	bucket := location.Data["bucket"]
+	key := location.Data["key"]
+
+	objectInfo, err := store.client.StatObject(context.Background(), bucket, key, minio.StatObjectOptions{})
+	if err != nil {
+		return 0, err
+	}
+
+	if objectInfo.Size > math.MaxInt32 {
+		return 0, fmt.Errorf("object size %d exceeds max int size", objectInfo.Size)
+	}
+
+	return int(objectInfo.Size), nil
+}
+
+func (store *MinIOArtifactStore) SaveResult(jobSpec common.JobSpec, data []byte) (ArtifactLocation, error) {
+	return store.saveArtifact(RESULTS_BUCKET_NAME, deriveKeyFromJobSpec(jobSpec), data, "application/zip")
+}
+
+func (store *MinIOArtifactStore) getArtifact(location ArtifactLocation) ([]byte, error) {
+	bucket := location.Data["bucket"]
+	key := location.Data["key"]
+
+	object, err := store.client.GetObject(context.Background(), bucket, key, minio.GetObjectOptions{})
+	if err != nil {
+		return nil, err
+	}
+	defer object.Close()
+
+	data, err := io.ReadAll(object)
+	if err != nil {
+		return nil, err
+	}
+
+	return data, nil
+}
+
+func (store *MinIOArtifactStore) saveArtifact(bucket, key string, data []byte, contentType string) (ArtifactLocation, error) {
+	_, err := store.client.PutObject(context.Background(), bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{
+		ContentType: contentType,
+	})
+	if err != nil {
+		return ArtifactLocation{}, err
+	}
+
+	location := ArtifactLocation{
+		Data: map[string]string{
+			"bucket": bucket,
+			"key":    key,
+		},
+	}
+
+	return location, nil
+}