mrva/mrvacommander
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replaced the dynamic table type ArtifactLocation with struct keys

Browse Source 
The original is present in comment form for reference
This commit is contained in:
Michael Hohn
2024-07-10 13:08:40 -07:00
committed by =Michael Hohn
parent 2df48b9f98
commit 8965725e42
4 changed files with 44 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.dockerignore Normal file
View File

@@ -0,0 +1,7 @@
# Excludes
/dbstore-data
/qpstore-data
/test-data
/venv
/client

6
pkg/artifactstore/common.go
View File

@@ -7,7 +7,7 @@ import (
// XX: static types: split by type?
// Restrict the keys / values for ArtifactLocation and centralize the common ones
// here
// here
const (
AF_VAL_BUCKET_RESULTS = "results"
AF_VAL_BUCKET_PACKS = "packs"
@@ -20,7 +20,9 @@ type ArtifactLocation struct {
// For example, a simple key-value pair could be "path" -> "/path/to/artifact.tgz".
// Alternatively, a more complex example could be "bucket" -> "example", "key" -> "UNIQUE_ARTIFACT_IDENTIFIER".
// XX: static types
Data map[string]string `json:"data"`
// Data map[string]string `json:"data"`
Key string // location in bucket
Bucket string // which bucket: packs or results
}
// deriveKeyFromSessionId generates a key for a query pack based on the job ID

30
pkg/artifactstore/store_memory.go
View File

@@ -25,7 +25,9 @@ func (store *InMemoryArtifactStore) GetQueryPack(location ArtifactLocation) ([]b
store.mu.Lock()
defer store.mu.Unlock()
key := location.Data[AF_KEY_KEY]
// XX: static types
// key := location.Data[AF_KEY_KEY]
key := location.Key
data, exists := store.packs[key]
if !exists {
return nil, fmt.Errorf("query pack not found: %s", key)
@@ -43,10 +45,12 @@ func (store *InMemoryArtifactStore) SaveQueryPack(sessionId int, data []byte) (A
// XX: static types
location := ArtifactLocation{
Data: map[string]string{
AF_KEY_BUCKET: AF_VAL_BUCKET_PACKS,
AF_KEY_KEY: key,
},
Bucket: AF_VAL_BUCKET_PACKS,
Key: key,
// Data: map[string]string{
// AF_KEY_BUCKET: AF_VAL_BUCKET_PACKS,
// AF_KEY_KEY: key,
// },
}
return location, nil
}
@@ -56,7 +60,8 @@ func (store *InMemoryArtifactStore) GetResult(location ArtifactLocation) ([]byte
store.mu.Lock()
defer store.mu.Unlock()
key := location.Data[AF_KEY_KEY]
// key := location.Data[AF_KEY_KEY]
key := location.Key
data, exists := store.results[key]
if !exists {
return nil, fmt.Errorf("result not found: %s", key)
@@ -69,7 +74,8 @@ func (store *InMemoryArtifactStore) GetResultSize(location ArtifactLocation) (in
store.mu.Lock()
defer store.mu.Unlock()
key := location.Data[AF_KEY_KEY]
// key := location.Data[AF_KEY_KEY]
key := location.Key
data, exists := store.results[key]
if !exists {
return 0, fmt.Errorf("result not found: %s", key)
@@ -87,10 +93,12 @@ func (store *InMemoryArtifactStore) SaveResult(jobSpec common.JobSpec, data []by
// XX: static types
location := ArtifactLocation{
Data: map[string]string{
AF_KEY_BUCKET: AF_VAL_BUCKET_RESULTS,
AF_KEY_KEY: key,
},
Bucket: AF_VAL_BUCKET_RESULTS,
Key: key,
// Data: map[string]string{
// AF_KEY_BUCKET: AF_VAL_BUCKET_RESULTS,
// AF_KEY_KEY: key,
// },
}
return location, nil
}

22
pkg/artifactstore/store_minio.go
View File

@@ -56,8 +56,10 @@ func (store *MinIOArtifactStore) GetResult(location ArtifactLocation) ([]byte, e
}
func (store *MinIOArtifactStore) GetResultSize(location ArtifactLocation) (int, error) {
bucket := location.Data[AF_KEY_BUCKET]
key := location.Data[AF_KEY_KEY]
// bucket := location.Data[AF_KEY_BUCKET]
// key := location.Data[AF_KEY_KEY]
bucket := location.Bucket
key := location.Key
objectInfo, err := store.client.StatObject(context.Background(), bucket, key, minio.StatObjectOptions{})
if err != nil {
@@ -76,8 +78,10 @@ func (store *MinIOArtifactStore) SaveResult(jobSpec common.JobSpec, data []byte)
}
func (store *MinIOArtifactStore) getArtifact(location ArtifactLocation) ([]byte, error) {
bucket := location.Data[AF_KEY_BUCKET]
key := location.Data[AF_KEY_KEY]
bucket := location.Bucket
key := location.Key
// bucket := location.Data[AF_KEY_BUCKET]
// key := location.Data[AF_KEY_KEY]
object, err := store.client.GetObject(context.Background(), bucket, key, minio.GetObjectOptions{})
if err != nil {
@@ -105,10 +109,12 @@ func (store *MinIOArtifactStore) saveArtifact(bucket, key string, data []byte,
// XX: static types
location := ArtifactLocation{
Data: map[string]string{
AF_KEY_BUCKET: bucket,
AF_KEY_KEY: key,
},
Bucket: bucket,
Key: key,
// Data: map[string]string{
// AF_KEY_BUCKET: bucket,
// AF_KEY_KEY: key,
// },
}
return location, nil