Type checking fix: Restrict the keys / values for ArtifactLocation and centralize the common ones
This commit is contained in:
committed by
=Michael Hohn
parent
b3cf7a4f65
commit
3566f5169e
@@ -5,10 +5,21 @@ import (
|
|||||||
"mrvacommander/pkg/common"
|
"mrvacommander/pkg/common"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// XX: static types: split by type?
|
||||||
|
// Restrict the keys / values for ArtifactLocation and centralize the common ones
|
||||||
|
// here
|
||||||
|
const (
|
||||||
|
AF_VAL_BUCKET_RESULTS = "results"
|
||||||
|
AF_VAL_BUCKET_PACKS = "packs"
|
||||||
|
AF_KEY_BUCKET = "bucket"
|
||||||
|
AF_KEY_KEY = "key"
|
||||||
|
)
|
||||||
|
|
||||||
type ArtifactLocation struct {
|
type ArtifactLocation struct {
|
||||||
// Data is a map of key-value pairs that describe the location of the artifact.
|
// Data is a map of key-value pairs that describe the location of the artifact.
|
||||||
// For example, a simple key-value pair could be "path" -> "/path/to/artifact.tgz".
|
// For example, a simple key-value pair could be "path" -> "/path/to/artifact.tgz".
|
||||||
// Alternatively, a more complex example could be "bucket" -> "example", "key" -> "UNIQUE_ARTIFACT_IDENTIFIER".
|
// Alternatively, a more complex example could be "bucket" -> "example", "key" -> "UNIQUE_ARTIFACT_IDENTIFIER".
|
||||||
|
// XX: static types
|
||||||
Data map[string]string `json:"data"`
|
Data map[string]string `json:"data"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ func (store *InMemoryArtifactStore) GetQueryPack(location ArtifactLocation) ([]b
|
|||||||
store.mu.Lock()
|
store.mu.Lock()
|
||||||
defer store.mu.Unlock()
|
defer store.mu.Unlock()
|
||||||
|
|
||||||
key := location.Data["key"]
|
key := location.Data[AF_KEY_KEY]
|
||||||
data, exists := store.packs[key]
|
data, exists := store.packs[key]
|
||||||
if !exists {
|
if !exists {
|
||||||
return nil, fmt.Errorf("query pack not found: %s", key)
|
return nil, fmt.Errorf("query pack not found: %s", key)
|
||||||
@@ -41,10 +41,11 @@ func (store *InMemoryArtifactStore) SaveQueryPack(sessionId int, data []byte) (A
|
|||||||
key := deriveKeyFromSessionId(sessionId)
|
key := deriveKeyFromSessionId(sessionId)
|
||||||
store.packs[key] = data
|
store.packs[key] = data
|
||||||
|
|
||||||
|
// XX: static types
|
||||||
location := ArtifactLocation{
|
location := ArtifactLocation{
|
||||||
Data: map[string]string{
|
Data: map[string]string{
|
||||||
"bucket": "packs",
|
AF_KEY_BUCKET: AF_VAL_BUCKET_PACKS,
|
||||||
"key": key,
|
AF_KEY_KEY: key,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
return location, nil
|
return location, nil
|
||||||
@@ -55,7 +56,7 @@ func (store *InMemoryArtifactStore) GetResult(location ArtifactLocation) ([]byte
|
|||||||
store.mu.Lock()
|
store.mu.Lock()
|
||||||
defer store.mu.Unlock()
|
defer store.mu.Unlock()
|
||||||
|
|
||||||
key := location.Data["key"]
|
key := location.Data[AF_KEY_KEY]
|
||||||
data, exists := store.results[key]
|
data, exists := store.results[key]
|
||||||
if !exists {
|
if !exists {
|
||||||
return nil, fmt.Errorf("result not found: %s", key)
|
return nil, fmt.Errorf("result not found: %s", key)
|
||||||
@@ -68,7 +69,7 @@ func (store *InMemoryArtifactStore) GetResultSize(location ArtifactLocation) (in
|
|||||||
store.mu.Lock()
|
store.mu.Lock()
|
||||||
defer store.mu.Unlock()
|
defer store.mu.Unlock()
|
||||||
|
|
||||||
key := location.Data["key"]
|
key := location.Data[AF_KEY_KEY]
|
||||||
data, exists := store.results[key]
|
data, exists := store.results[key]
|
||||||
if !exists {
|
if !exists {
|
||||||
return 0, fmt.Errorf("result not found: %s", key)
|
return 0, fmt.Errorf("result not found: %s", key)
|
||||||
@@ -84,10 +85,11 @@ func (store *InMemoryArtifactStore) SaveResult(jobSpec common.JobSpec, data []by
|
|||||||
key := deriveKeyFromJobSpec(jobSpec)
|
key := deriveKeyFromJobSpec(jobSpec)
|
||||||
store.results[key] = data
|
store.results[key] = data
|
||||||
|
|
||||||
|
// XX: static types
|
||||||
location := ArtifactLocation{
|
location := ArtifactLocation{
|
||||||
Data: map[string]string{
|
Data: map[string]string{
|
||||||
"bucket": "results",
|
AF_KEY_BUCKET: AF_VAL_BUCKET_RESULTS,
|
||||||
"key": key,
|
AF_KEY_KEY: key,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
return location, nil
|
return location, nil
|
||||||
|
|||||||
@@ -13,11 +13,6 @@ import (
|
|||||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
|
||||||
RESULTS_BUCKET_NAME = "results"
|
|
||||||
PACKS_BUCKET_NAME = "packs"
|
|
||||||
)
|
|
||||||
|
|
||||||
type MinIOArtifactStore struct {
|
type MinIOArtifactStore struct {
|
||||||
client *minio.Client
|
client *minio.Client
|
||||||
}
|
}
|
||||||
@@ -34,12 +29,12 @@ func NewMinIOArtifactStore(endpoint, id, secret string) (*MinIOArtifactStore, er
|
|||||||
slog.Info("Connected to MinIO artifact store server")
|
slog.Info("Connected to MinIO artifact store server")
|
||||||
|
|
||||||
// Create "results" bucket
|
// Create "results" bucket
|
||||||
if err := common.CreateMinIOBucketIfNotExists(minioClient, RESULTS_BUCKET_NAME); err != nil {
|
if err := common.CreateMinIOBucketIfNotExists(minioClient, AF_VAL_BUCKET_RESULTS); err != nil {
|
||||||
return nil, fmt.Errorf("could not create results bucket: %v", err)
|
return nil, fmt.Errorf("could not create results bucket: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create "packs" bucket
|
// Create "packs" bucket
|
||||||
if err := common.CreateMinIOBucketIfNotExists(minioClient, PACKS_BUCKET_NAME); err != nil {
|
if err := common.CreateMinIOBucketIfNotExists(minioClient, AF_VAL_BUCKET_PACKS); err != nil {
|
||||||
return nil, fmt.Errorf("could not create packs bucket: %v", err)
|
return nil, fmt.Errorf("could not create packs bucket: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -53,7 +48,7 @@ func (store *MinIOArtifactStore) GetQueryPack(location ArtifactLocation) ([]byte
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (store *MinIOArtifactStore) SaveQueryPack(jobId int, data []byte) (ArtifactLocation, error) {
|
func (store *MinIOArtifactStore) SaveQueryPack(jobId int, data []byte) (ArtifactLocation, error) {
|
||||||
return store.saveArtifact(PACKS_BUCKET_NAME, deriveKeyFromSessionId(jobId), data, "application/gzip")
|
return store.saveArtifact(AF_VAL_BUCKET_PACKS, deriveKeyFromSessionId(jobId), data, "application/gzip")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (store *MinIOArtifactStore) GetResult(location ArtifactLocation) ([]byte, error) {
|
func (store *MinIOArtifactStore) GetResult(location ArtifactLocation) ([]byte, error) {
|
||||||
@@ -61,8 +56,8 @@ func (store *MinIOArtifactStore) GetResult(location ArtifactLocation) ([]byte, e
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (store *MinIOArtifactStore) GetResultSize(location ArtifactLocation) (int, error) {
|
func (store *MinIOArtifactStore) GetResultSize(location ArtifactLocation) (int, error) {
|
||||||
bucket := location.Data["bucket"]
|
bucket := location.Data[AF_KEY_BUCKET]
|
||||||
key := location.Data["key"]
|
key := location.Data[AF_KEY_KEY]
|
||||||
|
|
||||||
objectInfo, err := store.client.StatObject(context.Background(), bucket, key, minio.StatObjectOptions{})
|
objectInfo, err := store.client.StatObject(context.Background(), bucket, key, minio.StatObjectOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -77,12 +72,12 @@ func (store *MinIOArtifactStore) GetResultSize(location ArtifactLocation) (int,
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (store *MinIOArtifactStore) SaveResult(jobSpec common.JobSpec, data []byte) (ArtifactLocation, error) {
|
func (store *MinIOArtifactStore) SaveResult(jobSpec common.JobSpec, data []byte) (ArtifactLocation, error) {
|
||||||
return store.saveArtifact(RESULTS_BUCKET_NAME, deriveKeyFromJobSpec(jobSpec), data, "application/zip")
|
return store.saveArtifact(AF_VAL_BUCKET_RESULTS, deriveKeyFromJobSpec(jobSpec), data, "application/zip")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (store *MinIOArtifactStore) getArtifact(location ArtifactLocation) ([]byte, error) {
|
func (store *MinIOArtifactStore) getArtifact(location ArtifactLocation) ([]byte, error) {
|
||||||
bucket := location.Data["bucket"]
|
bucket := location.Data[AF_KEY_BUCKET]
|
||||||
key := location.Data["key"]
|
key := location.Data[AF_KEY_KEY]
|
||||||
|
|
||||||
object, err := store.client.GetObject(context.Background(), bucket, key, minio.GetObjectOptions{})
|
object, err := store.client.GetObject(context.Background(), bucket, key, minio.GetObjectOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -98,18 +93,21 @@ func (store *MinIOArtifactStore) getArtifact(location ArtifactLocation) ([]byte,
|
|||||||
return data, nil
|
return data, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (store *MinIOArtifactStore) saveArtifact(bucket, key string, data []byte, contentType string) (ArtifactLocation, error) {
|
func (store *MinIOArtifactStore) saveArtifact(bucket, key string, data []byte,
|
||||||
_, err := store.client.PutObject(context.Background(), bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{
|
contentType string) (ArtifactLocation, error) {
|
||||||
ContentType: contentType,
|
_, err := store.client.PutObject(context.Background(), bucket, key,
|
||||||
})
|
bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{
|
||||||
|
ContentType: contentType,
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ArtifactLocation{}, err
|
return ArtifactLocation{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// XX: static types
|
||||||
location := ArtifactLocation{
|
location := ArtifactLocation{
|
||||||
Data: map[string]string{
|
Data: map[string]string{
|
||||||
"bucket": bucket,
|
AF_KEY_BUCKET: bucket,
|
||||||
"key": key,
|
AF_KEY_KEY: key,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -26,8 +26,10 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type SarifRun struct {
|
type SarifRun struct {
|
||||||
|
// XX: static types
|
||||||
VersionControlProvenance []interface{} `json:"versionControlProvenance,omitempty"`
|
VersionControlProvenance []interface{} `json:"versionControlProvenance,omitempty"`
|
||||||
Results []interface{} `json:"results"`
|
// XX: never set, only read
|
||||||
|
Results []interface{} `json:"results"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type Sarif struct {
|
type Sarif struct {
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ type CodeQLDatabaseLocation struct {
|
|||||||
// `data` is a map of key-value pairs that describe the location of the database.
|
// `data` is a map of key-value pairs that describe the location of the database.
|
||||||
// For example, a simple key-value pair could be "path" -> "/path/to/database.zip".
|
// For example, a simple key-value pair could be "path" -> "/path/to/database.zip".
|
||||||
// A more complex implementation could be "bucket" -> "example", "key" -> "unique_identifier".
|
// A more complex implementation could be "bucket" -> "example", "key" -> "unique_identifier".
|
||||||
|
// XX: static types
|
||||||
data map[string]string
|
data map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"mrvacommander/pkg/artifactstore"
|
||||||
"mrvacommander/pkg/common"
|
"mrvacommander/pkg/common"
|
||||||
|
|
||||||
"github.com/minio/minio-go/v7"
|
"github.com/minio/minio-go/v7"
|
||||||
@@ -58,8 +59,8 @@ func (store *MinIOCodeQLDatabaseStore) FindAvailableDBs(analysisReposRequested [
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (store *MinIOCodeQLDatabaseStore) GetDatabase(location CodeQLDatabaseLocation) ([]byte, error) {
|
func (store *MinIOCodeQLDatabaseStore) GetDatabase(location CodeQLDatabaseLocation) ([]byte, error) {
|
||||||
bucket := location.data["bucket"]
|
bucket := location.data[artifactstore.AF_KEY_BUCKET]
|
||||||
key := location.data["key"]
|
key := location.data[artifactstore.AF_KEY_KEY]
|
||||||
|
|
||||||
object, err := store.client.GetObject(context.Background(), bucket, key, minio.GetObjectOptions{})
|
object, err := store.client.GetObject(context.Background(), bucket, key, minio.GetObjectOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ func NewRabbitMQQueue(
|
|||||||
const (
|
const (
|
||||||
tryCount = 5
|
tryCount = 5
|
||||||
retryDelaySec = 3
|
retryDelaySec = 3
|
||||||
|
// XX: static typing?
|
||||||
jobsQueueName = "tasks"
|
jobsQueueName = "tasks"
|
||||||
resultsQueueName = "results"
|
resultsQueueName = "results"
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user