Type checking fix: Restrict the keys / values for ArtifactLocation and centralize the common ones

This commit is contained in:
Michael Hohn
2024-07-08 12:07:46 -07:00
committed by =Michael Hohn
parent b3cf7a4f65
commit 3566f5169e
7 changed files with 45 additions and 29 deletions

View File

@@ -5,10 +5,21 @@ import (
"mrvacommander/pkg/common" "mrvacommander/pkg/common"
) )
// XX: static types: split by type?
// Restrict the keys / values for ArtifactLocation and centralize the common ones
// here
const (
AF_VAL_BUCKET_RESULTS = "results"
AF_VAL_BUCKET_PACKS = "packs"
AF_KEY_BUCKET = "bucket"
AF_KEY_KEY = "key"
)
type ArtifactLocation struct { type ArtifactLocation struct {
// Data is a map of key-value pairs that describe the location of the artifact. // Data is a map of key-value pairs that describe the location of the artifact.
// For example, a simple key-value pair could be "path" -> "/path/to/artifact.tgz". // For example, a simple key-value pair could be "path" -> "/path/to/artifact.tgz".
// Alternatively, a more complex example could be "bucket" -> "example", "key" -> "UNIQUE_ARTIFACT_IDENTIFIER". // Alternatively, a more complex example could be "bucket" -> "example", "key" -> "UNIQUE_ARTIFACT_IDENTIFIER".
// XX: static types
Data map[string]string `json:"data"` Data map[string]string `json:"data"`
} }

View File

@@ -25,7 +25,7 @@ func (store *InMemoryArtifactStore) GetQueryPack(location ArtifactLocation) ([]b
store.mu.Lock() store.mu.Lock()
defer store.mu.Unlock() defer store.mu.Unlock()
key := location.Data["key"] key := location.Data[AF_KEY_KEY]
data, exists := store.packs[key] data, exists := store.packs[key]
if !exists { if !exists {
return nil, fmt.Errorf("query pack not found: %s", key) return nil, fmt.Errorf("query pack not found: %s", key)
@@ -41,10 +41,11 @@ func (store *InMemoryArtifactStore) SaveQueryPack(sessionId int, data []byte) (A
key := deriveKeyFromSessionId(sessionId) key := deriveKeyFromSessionId(sessionId)
store.packs[key] = data store.packs[key] = data
// XX: static types
location := ArtifactLocation{ location := ArtifactLocation{
Data: map[string]string{ Data: map[string]string{
"bucket": "packs", AF_KEY_BUCKET: AF_VAL_BUCKET_PACKS,
"key": key, AF_KEY_KEY: key,
}, },
} }
return location, nil return location, nil
@@ -55,7 +56,7 @@ func (store *InMemoryArtifactStore) GetResult(location ArtifactLocation) ([]byte
store.mu.Lock() store.mu.Lock()
defer store.mu.Unlock() defer store.mu.Unlock()
key := location.Data["key"] key := location.Data[AF_KEY_KEY]
data, exists := store.results[key] data, exists := store.results[key]
if !exists { if !exists {
return nil, fmt.Errorf("result not found: %s", key) return nil, fmt.Errorf("result not found: %s", key)
@@ -68,7 +69,7 @@ func (store *InMemoryArtifactStore) GetResultSize(location ArtifactLocation) (in
store.mu.Lock() store.mu.Lock()
defer store.mu.Unlock() defer store.mu.Unlock()
key := location.Data["key"] key := location.Data[AF_KEY_KEY]
data, exists := store.results[key] data, exists := store.results[key]
if !exists { if !exists {
return 0, fmt.Errorf("result not found: %s", key) return 0, fmt.Errorf("result not found: %s", key)
@@ -84,10 +85,11 @@ func (store *InMemoryArtifactStore) SaveResult(jobSpec common.JobSpec, data []by
key := deriveKeyFromJobSpec(jobSpec) key := deriveKeyFromJobSpec(jobSpec)
store.results[key] = data store.results[key] = data
// XX: static types
location := ArtifactLocation{ location := ArtifactLocation{
Data: map[string]string{ Data: map[string]string{
"bucket": "results", AF_KEY_BUCKET: AF_VAL_BUCKET_RESULTS,
"key": key, AF_KEY_KEY: key,
}, },
} }
return location, nil return location, nil

View File

@@ -13,11 +13,6 @@ import (
"github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/credentials"
) )
const (
RESULTS_BUCKET_NAME = "results"
PACKS_BUCKET_NAME = "packs"
)
type MinIOArtifactStore struct { type MinIOArtifactStore struct {
client *minio.Client client *minio.Client
} }
@@ -34,12 +29,12 @@ func NewMinIOArtifactStore(endpoint, id, secret string) (*MinIOArtifactStore, er
slog.Info("Connected to MinIO artifact store server") slog.Info("Connected to MinIO artifact store server")
// Create "results" bucket // Create "results" bucket
if err := common.CreateMinIOBucketIfNotExists(minioClient, RESULTS_BUCKET_NAME); err != nil { if err := common.CreateMinIOBucketIfNotExists(minioClient, AF_VAL_BUCKET_RESULTS); err != nil {
return nil, fmt.Errorf("could not create results bucket: %v", err) return nil, fmt.Errorf("could not create results bucket: %v", err)
} }
// Create "packs" bucket // Create "packs" bucket
if err := common.CreateMinIOBucketIfNotExists(minioClient, PACKS_BUCKET_NAME); err != nil { if err := common.CreateMinIOBucketIfNotExists(minioClient, AF_VAL_BUCKET_PACKS); err != nil {
return nil, fmt.Errorf("could not create packs bucket: %v", err) return nil, fmt.Errorf("could not create packs bucket: %v", err)
} }
@@ -53,7 +48,7 @@ func (store *MinIOArtifactStore) GetQueryPack(location ArtifactLocation) ([]byte
} }
func (store *MinIOArtifactStore) SaveQueryPack(jobId int, data []byte) (ArtifactLocation, error) { func (store *MinIOArtifactStore) SaveQueryPack(jobId int, data []byte) (ArtifactLocation, error) {
return store.saveArtifact(PACKS_BUCKET_NAME, deriveKeyFromSessionId(jobId), data, "application/gzip") return store.saveArtifact(AF_VAL_BUCKET_PACKS, deriveKeyFromSessionId(jobId), data, "application/gzip")
} }
func (store *MinIOArtifactStore) GetResult(location ArtifactLocation) ([]byte, error) { func (store *MinIOArtifactStore) GetResult(location ArtifactLocation) ([]byte, error) {
@@ -61,8 +56,8 @@ func (store *MinIOArtifactStore) GetResult(location ArtifactLocation) ([]byte, e
} }
func (store *MinIOArtifactStore) GetResultSize(location ArtifactLocation) (int, error) { func (store *MinIOArtifactStore) GetResultSize(location ArtifactLocation) (int, error) {
bucket := location.Data["bucket"] bucket := location.Data[AF_KEY_BUCKET]
key := location.Data["key"] key := location.Data[AF_KEY_KEY]
objectInfo, err := store.client.StatObject(context.Background(), bucket, key, minio.StatObjectOptions{}) objectInfo, err := store.client.StatObject(context.Background(), bucket, key, minio.StatObjectOptions{})
if err != nil { if err != nil {
@@ -77,12 +72,12 @@ func (store *MinIOArtifactStore) GetResultSize(location ArtifactLocation) (int,
} }
func (store *MinIOArtifactStore) SaveResult(jobSpec common.JobSpec, data []byte) (ArtifactLocation, error) { func (store *MinIOArtifactStore) SaveResult(jobSpec common.JobSpec, data []byte) (ArtifactLocation, error) {
return store.saveArtifact(RESULTS_BUCKET_NAME, deriveKeyFromJobSpec(jobSpec), data, "application/zip") return store.saveArtifact(AF_VAL_BUCKET_RESULTS, deriveKeyFromJobSpec(jobSpec), data, "application/zip")
} }
func (store *MinIOArtifactStore) getArtifact(location ArtifactLocation) ([]byte, error) { func (store *MinIOArtifactStore) getArtifact(location ArtifactLocation) ([]byte, error) {
bucket := location.Data["bucket"] bucket := location.Data[AF_KEY_BUCKET]
key := location.Data["key"] key := location.Data[AF_KEY_KEY]
object, err := store.client.GetObject(context.Background(), bucket, key, minio.GetObjectOptions{}) object, err := store.client.GetObject(context.Background(), bucket, key, minio.GetObjectOptions{})
if err != nil { if err != nil {
@@ -98,18 +93,21 @@ func (store *MinIOArtifactStore) getArtifact(location ArtifactLocation) ([]byte,
return data, nil return data, nil
} }
func (store *MinIOArtifactStore) saveArtifact(bucket, key string, data []byte, contentType string) (ArtifactLocation, error) { func (store *MinIOArtifactStore) saveArtifact(bucket, key string, data []byte,
_, err := store.client.PutObject(context.Background(), bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ contentType string) (ArtifactLocation, error) {
ContentType: contentType, _, err := store.client.PutObject(context.Background(), bucket, key,
}) bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{
ContentType: contentType,
})
if err != nil { if err != nil {
return ArtifactLocation{}, err return ArtifactLocation{}, err
} }
// XX: static types
location := ArtifactLocation{ location := ArtifactLocation{
Data: map[string]string{ Data: map[string]string{
"bucket": bucket, AF_KEY_BUCKET: bucket,
"key": key, AF_KEY_KEY: key,
}, },
} }

View File

@@ -26,8 +26,10 @@ const (
) )
type SarifRun struct { type SarifRun struct {
// XX: static types
VersionControlProvenance []interface{} `json:"versionControlProvenance,omitempty"` VersionControlProvenance []interface{} `json:"versionControlProvenance,omitempty"`
Results []interface{} `json:"results"` // XX: never set, only read
Results []interface{} `json:"results"`
} }
type Sarif struct { type Sarif struct {

View File

@@ -8,6 +8,7 @@ type CodeQLDatabaseLocation struct {
// `data` is a map of key-value pairs that describe the location of the database. // `data` is a map of key-value pairs that describe the location of the database.
// For example, a simple key-value pair could be "path" -> "/path/to/database.zip". // For example, a simple key-value pair could be "path" -> "/path/to/database.zip".
// A more complex implementation could be "bucket" -> "example", "key" -> "unique_identifier". // A more complex implementation could be "bucket" -> "example", "key" -> "unique_identifier".
// XX: static types
data map[string]string data map[string]string
} }

View File

@@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"io" "io"
"log/slog" "log/slog"
"mrvacommander/pkg/artifactstore"
"mrvacommander/pkg/common" "mrvacommander/pkg/common"
"github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7"
@@ -58,8 +59,8 @@ func (store *MinIOCodeQLDatabaseStore) FindAvailableDBs(analysisReposRequested [
} }
func (store *MinIOCodeQLDatabaseStore) GetDatabase(location CodeQLDatabaseLocation) ([]byte, error) { func (store *MinIOCodeQLDatabaseStore) GetDatabase(location CodeQLDatabaseLocation) ([]byte, error) {
bucket := location.data["bucket"] bucket := location.data[artifactstore.AF_KEY_BUCKET]
key := location.data["key"] key := location.data[artifactstore.AF_KEY_KEY]
object, err := store.client.GetObject(context.Background(), bucket, key, minio.GetObjectOptions{}) object, err := store.client.GetObject(context.Background(), bucket, key, minio.GetObjectOptions{})
if err != nil { if err != nil {

View File

@@ -34,6 +34,7 @@ func NewRabbitMQQueue(
const ( const (
tryCount = 5 tryCount = 5
retryDelaySec = 3 retryDelaySec = 3
// XX: static typing?
jobsQueueName = "tasks" jobsQueueName = "tasks"
resultsQueueName = "results" resultsQueueName = "results"
) )