mrva/mrvacommander
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replaced the dynamic table type ArtifactLocation with struct keys

Browse Source 
The original is present in comment form for reference
This commit is contained in:
Michael Hohn
2024-07-10 13:08:40 -07:00
committed by =Michael Hohn
parent 2df48b9f98
commit 8965725e42
4 changed files with 44 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.dockerignore Normal file
View File

@@ -0,0 +1,7 @@
# Excludes
/dbstore-data
/qpstore-data
/test-data
/venv
/client

4
pkg/artifactstore/common.go
View File

@@ -20,7 +20,9 @@ type ArtifactLocation struct {
// For example, a simple key-value pair could be "path" -> "/path/to/artifact.tgz". // For example, a simple key-value pair could be "path" -> "/path/to/artifact.tgz".
// Alternatively, a more complex example could be "bucket" -> "example", "key" -> "UNIQUE_ARTIFACT_IDENTIFIER". // Alternatively, a more complex example could be "bucket" -> "example", "key" -> "UNIQUE_ARTIFACT_IDENTIFIER".
// XX: static types // XX: static types
Data map[string]string `json:"data"` // Data map[string]string `json:"data"`
Key string // location in bucket
Bucket string // which bucket: packs or results
} }
// deriveKeyFromSessionId generates a key for a query pack based on the job ID // deriveKeyFromSessionId generates a key for a query pack based on the job ID

30
pkg/artifactstore/store_memory.go
View File

@@ -25,7 +25,9 @@ func (store *InMemoryArtifactStore) GetQueryPack(location ArtifactLocation) ([]b
store.mu.Lock() store.mu.Lock()
defer store.mu.Unlock() defer store.mu.Unlock()
key := location.Data[AF_KEY_KEY] // XX: static types
// key := location.Data[AF_KEY_KEY]
key := location.Key
data, exists := store.packs[key] data, exists := store.packs[key]
if !exists { if !exists {
return nil, fmt.Errorf("query pack not found: %s", key) return nil, fmt.Errorf("query pack not found: %s", key)
@@ -43,10 +45,12 @@ func (store *InMemoryArtifactStore) SaveQueryPack(sessionId int, data []byte) (A
// XX: static types // XX: static types
location := ArtifactLocation{ location := ArtifactLocation{
Data: map[string]string{ Bucket: AF_VAL_BUCKET_PACKS,
AF_KEY_BUCKET: AF_VAL_BUCKET_PACKS, Key: key,
AF_KEY_KEY: key, // Data: map[string]string{
}, // AF_KEY_BUCKET: AF_VAL_BUCKET_PACKS,
// AF_KEY_KEY: key,
// },
} }
return location, nil return location, nil
} }
@@ -56,7 +60,8 @@ func (store *InMemoryArtifactStore) GetResult(location ArtifactLocation) ([]byte
store.mu.Lock() store.mu.Lock()
defer store.mu.Unlock() defer store.mu.Unlock()
key := location.Data[AF_KEY_KEY] // key := location.Data[AF_KEY_KEY]
key := location.Key
data, exists := store.results[key] data, exists := store.results[key]
if !exists { if !exists {
return nil, fmt.Errorf("result not found: %s", key) return nil, fmt.Errorf("result not found: %s", key)
@@ -69,7 +74,8 @@ func (store *InMemoryArtifactStore) GetResultSize(location ArtifactLocation) (in
store.mu.Lock() store.mu.Lock()
defer store.mu.Unlock() defer store.mu.Unlock()
key := location.Data[AF_KEY_KEY] // key := location.Data[AF_KEY_KEY]
key := location.Key
data, exists := store.results[key] data, exists := store.results[key]
if !exists { if !exists {
return 0, fmt.Errorf("result not found: %s", key) return 0, fmt.Errorf("result not found: %s", key)
@@ -87,10 +93,12 @@ func (store *InMemoryArtifactStore) SaveResult(jobSpec common.JobSpec, data []by
// XX: static types // XX: static types
location := ArtifactLocation{ location := ArtifactLocation{
Data: map[string]string{ Bucket: AF_VAL_BUCKET_RESULTS,
AF_KEY_BUCKET: AF_VAL_BUCKET_RESULTS, Key: key,
AF_KEY_KEY: key, // Data: map[string]string{
}, // AF_KEY_BUCKET: AF_VAL_BUCKET_RESULTS,
// AF_KEY_KEY: key,
// },
} }
return location, nil return location, nil
} }

22
pkg/artifactstore/store_minio.go
View File

@@ -56,8 +56,10 @@ func (store *MinIOArtifactStore) GetResult(location ArtifactLocation) ([]byte, e
} }
func (store *MinIOArtifactStore) GetResultSize(location ArtifactLocation) (int, error) { func (store *MinIOArtifactStore) GetResultSize(location ArtifactLocation) (int, error) {
bucket := location.Data[AF_KEY_BUCKET] // bucket := location.Data[AF_KEY_BUCKET]
key := location.Data[AF_KEY_KEY] // key := location.Data[AF_KEY_KEY]
bucket := location.Bucket
key := location.Key
objectInfo, err := store.client.StatObject(context.Background(), bucket, key, minio.StatObjectOptions{}) objectInfo, err := store.client.StatObject(context.Background(), bucket, key, minio.StatObjectOptions{})
if err != nil { if err != nil {
@@ -76,8 +78,10 @@ func (store *MinIOArtifactStore) SaveResult(jobSpec common.JobSpec, data []byte)
} }
func (store *MinIOArtifactStore) getArtifact(location ArtifactLocation) ([]byte, error) { func (store *MinIOArtifactStore) getArtifact(location ArtifactLocation) ([]byte, error) {
bucket := location.Data[AF_KEY_BUCKET] bucket := location.Bucket
key := location.Data[AF_KEY_KEY] key := location.Key
// bucket := location.Data[AF_KEY_BUCKET]
// key := location.Data[AF_KEY_KEY]
object, err := store.client.GetObject(context.Background(), bucket, key, minio.GetObjectOptions{}) object, err := store.client.GetObject(context.Background(), bucket, key, minio.GetObjectOptions{})
if err != nil { if err != nil {
@@ -105,10 +109,12 @@ func (store *MinIOArtifactStore) saveArtifact(bucket, key string, data []byte,
// XX: static types // XX: static types
location := ArtifactLocation{ location := ArtifactLocation{
Data: map[string]string{ Bucket: bucket,
AF_KEY_BUCKET: bucket, Key: key,
AF_KEY_KEY: key, // Data: map[string]string{
}, // AF_KEY_BUCKET: bucket,
// AF_KEY_KEY: key,
// },
} }
return location, nil return location, nil