mrva/mrvacommander
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update s3 endpoint handling

Browse Source 
virtual host extracts bucket name from endpoint environment variable;
path uses fixed bucket name
This commit is contained in:
Michael Hohn
2025-05-13 10:47:54 -07:00
committed by =Michael Hohn
parent bde8ac2db7
commit 5bdbd60cc5
5 changed files with 62 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/artifactstore/common.go
View File

@@ -8,7 +8,7 @@ import (
// Restrict the keys / values for ArtifactLocation and centralize the common ones
// here
const (
var (
AF_BUCKETNAME_RESULTS = "mrvabucket"
AF_BUCKETNAME_PACKS = "mrvabucket"
)

4
pkg/artifactstore/store_minio.go
View File

@@ -18,11 +18,11 @@ type MinIOArtifactStore struct {
client *minio.Client
}
func NewMinIOArtifactStore(endpoint, id, secret string) (*MinIOArtifactStore, error) {
func NewMinIOArtifactStore(endpoint, id, secret string, lookup minio.BucketLookupType) (*MinIOArtifactStore, error) {
minioClient, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(id, secret, ""),
Secure: false,
BucketLookup: minio.BucketLookupDNS, // Enable virtual-host-style addressing
BucketLookup: lookup,
})
if err != nil {
return nil, err

1
pkg/common/minio.go
View File

@@ -16,6 +16,7 @@ func CreateMinIOBucketIfNotExists(client *minio.Client, bucketName string) error
}
if !exists {
// if env.Get("MRVA_S3_PATHSTYLE") == "true" {}
slog.Info("Creating bucket", "name", bucketName)
err = client.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{})
if err != nil {

56
pkg/deploy/init.go
View File

@@ -4,12 +4,15 @@ import (
"fmt"
"log"
"log/slog"
"net/url"
"os"
"strconv"
"strings"
"github.com/hohn/mrvacommander/pkg/artifactstore"
"github.com/hohn/mrvacommander/pkg/qldbstore"
"github.com/hohn/mrvacommander/pkg/queue"
"github.com/minio/minio-go/v7"
)
func validateEnvVars(requiredEnvVars []string) {
@@ -61,20 +64,43 @@ func InitMinIOArtifactStore() (artifactstore.Store, error) {
"ARTIFACT_MINIO_ENDPOINT",
"ARTIFACT_MINIO_ID",
"ARTIFACT_MINIO_SECRET",
"MRVA_MINIO_VIRTUAL_HOST",
}
validateEnvVars(requiredEnvVars)
endpoint := os.Getenv("ARTIFACT_MINIO_ENDPOINT")
id := os.Getenv("ARTIFACT_MINIO_ID")
secret := os.Getenv("ARTIFACT_MINIO_SECRET")
useVirtual := os.Getenv("MRVA_MINIO_VIRTUAL_HOST") == "1"
store, err := artifactstore.NewMinIOArtifactStore(endpoint, id, secret)
var lookup minio.BucketLookupType
var bucketName string
if useVirtual {
parsedURL, err := url.Parse(endpoint)
if err != nil {
return nil, fmt.Errorf("failed to parse ARTIFACT_MINIO_ENDPOINT: %w", err)
}
hostParts := strings.Split(parsedURL.Hostname(), ".")
if len(hostParts) < 2 {
return nil, fmt.Errorf("unable to extract bucket from host: %s", parsedURL.Hostname())
}
bucketName = hostParts[0]
lookup = minio.BucketLookupDNS
} else {
bucketName = "mrvabucket"
lookup = minio.BucketLookupPath
}
// TODO: unify into one. clean up state handling.
artifactstore.AF_BUCKETNAME_RESULTS = bucketName
artifactstore.AF_BUCKETNAME_PACKS = bucketName
store, err := artifactstore.NewMinIOArtifactStore(endpoint, id, secret, lookup)
if err != nil {
return nil, fmt.Errorf("failed to initialize artifact store: %v", err)
}
return store, nil
}
func InitMinIOCodeQLDatabaseStore() (qldbstore.Store, error) {
@@ -82,14 +108,38 @@ func InitMinIOCodeQLDatabaseStore() (qldbstore.Store, error) {
"QLDB_MINIO_ENDPOINT",
"QLDB_MINIO_ID",
"QLDB_MINIO_SECRET",
"MRVA_MINIO_VIRTUAL_HOST",
}
validateEnvVars(requiredEnvVars)
endpoint := os.Getenv("QLDB_MINIO_ENDPOINT")
id := os.Getenv("QLDB_MINIO_ID")
secret := os.Getenv("QLDB_MINIO_SECRET")
useVirtual := os.Getenv("MRVA_MINIO_VIRTUAL_HOST") == "1"
store, err := qldbstore.NewMinIOCodeQLDatabaseStore(endpoint, id, secret)
var lookup minio.BucketLookupType
var bucketName string
if useVirtual {
parsedURL, err := url.Parse(endpoint)
if err != nil {
return nil, fmt.Errorf("failed to parse QLDB_MINIO_ENDPOINT: %w", err)
}
hostParts := strings.Split(parsedURL.Hostname(), ".")
if len(hostParts) < 2 {
return nil, fmt.Errorf("unable to extract bucket from host: %s", parsedURL.Hostname())
}
bucketName = hostParts[0]
lookup = minio.BucketLookupDNS
} else {
bucketName = "mrvabucket"
lookup = minio.BucketLookupPath
}
// TODO: unify into one. clean up state handling.
qldbstore.QL_DB_BUCKETNAME = bucketName
store, err := qldbstore.NewMinIOCodeQLDatabaseStore(endpoint, id, secret, lookup)
if err != nil {
return nil, fmt.Errorf("failed to initialize ql database storage: %v", err)
}

8
pkg/qldbstore/qldbstore_minio.go
View File

@@ -14,7 +14,7 @@ import (
// XX: static types: split by type?
// Restrict the keys / values and centralize the common ones here
const (
var (
QL_DB_BUCKETNAME = "mrvabucket"
)
@@ -23,11 +23,13 @@ type MinIOCodeQLDatabaseStore struct {
bucketName string
}
func NewMinIOCodeQLDatabaseStore(endpoint, id, secret string) (*MinIOCodeQLDatabaseStore, error) {
func NewMinIOCodeQLDatabaseStore(endpoint, id, secret string,
lookup minio.BucketLookupType) (*MinIOCodeQLDatabaseStore, error) {
minioClient, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(id, secret, ""),
Secure: false,
BucketLookup: minio.BucketLookupDNS, // Enable virtual-host-style addressing
BucketLookup: lookup,
})
if err != nil {
return nil, err