From 5bdbd60cc53d8a9a6e2c2a8c67551bf8142deae2 Mon Sep 17 00:00:00 2001 From: Michael Hohn Date: Tue, 13 May 2025 10:47:54 -0700 Subject: [PATCH] update s3 endpoint handling virtual host extracts bucket name from endpoint environment variable; path uses fixed bucket name --- pkg/artifactstore/common.go | 2 +- pkg/artifactstore/store_minio.go | 4 +-- pkg/common/minio.go | 1 + pkg/deploy/init.go | 56 ++++++++++++++++++++++++++++++-- pkg/qldbstore/qldbstore_minio.go | 8 +++-- 5 files changed, 62 insertions(+), 9 deletions(-) diff --git a/pkg/artifactstore/common.go b/pkg/artifactstore/common.go index c66a015..7711519 100644 --- a/pkg/artifactstore/common.go +++ b/pkg/artifactstore/common.go @@ -8,7 +8,7 @@ import ( // Restrict the keys / values for ArtifactLocation and centralize the common ones // here -const ( +var ( AF_BUCKETNAME_RESULTS = "mrvabucket" AF_BUCKETNAME_PACKS = "mrvabucket" ) diff --git a/pkg/artifactstore/store_minio.go b/pkg/artifactstore/store_minio.go index 42dfa4d..992987c 100644 --- a/pkg/artifactstore/store_minio.go +++ b/pkg/artifactstore/store_minio.go @@ -18,11 +18,11 @@ type MinIOArtifactStore struct { client *minio.Client } -func NewMinIOArtifactStore(endpoint, id, secret string) (*MinIOArtifactStore, error) { +func NewMinIOArtifactStore(endpoint, id, secret string, lookup minio.BucketLookupType) (*MinIOArtifactStore, error) { minioClient, err := minio.New(endpoint, &minio.Options{ Creds: credentials.NewStaticV4(id, secret, ""), Secure: false, - BucketLookup: minio.BucketLookupDNS, // Enable virtual-host-style addressing + BucketLookup: lookup, }) if err != nil { return nil, err diff --git a/pkg/common/minio.go b/pkg/common/minio.go index ae7a154..96bb2cf 100644 --- a/pkg/common/minio.go +++ b/pkg/common/minio.go @@ -16,6 +16,7 @@ func CreateMinIOBucketIfNotExists(client *minio.Client, bucketName string) error } if !exists { + // if env.Get("MRVA_S3_PATHSTYLE") == "true" {} slog.Info("Creating bucket", "name", bucketName) err = client.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{}) if err != nil { diff --git a/pkg/deploy/init.go b/pkg/deploy/init.go index ec8e3f4..9d6a309 100644 --- a/pkg/deploy/init.go +++ b/pkg/deploy/init.go @@ -4,12 +4,15 @@ import ( "fmt" "log" "log/slog" + "net/url" "os" "strconv" + "strings" "github.com/hohn/mrvacommander/pkg/artifactstore" "github.com/hohn/mrvacommander/pkg/qldbstore" "github.com/hohn/mrvacommander/pkg/queue" + "github.com/minio/minio-go/v7" ) func validateEnvVars(requiredEnvVars []string) { @@ -61,20 +64,43 @@ func InitMinIOArtifactStore() (artifactstore.Store, error) { "ARTIFACT_MINIO_ENDPOINT", "ARTIFACT_MINIO_ID", "ARTIFACT_MINIO_SECRET", + "MRVA_MINIO_VIRTUAL_HOST", } validateEnvVars(requiredEnvVars) endpoint := os.Getenv("ARTIFACT_MINIO_ENDPOINT") id := os.Getenv("ARTIFACT_MINIO_ID") secret := os.Getenv("ARTIFACT_MINIO_SECRET") + useVirtual := os.Getenv("MRVA_MINIO_VIRTUAL_HOST") == "1" - store, err := artifactstore.NewMinIOArtifactStore(endpoint, id, secret) + var lookup minio.BucketLookupType + var bucketName string + if useVirtual { + parsedURL, err := url.Parse(endpoint) + if err != nil { + return nil, fmt.Errorf("failed to parse ARTIFACT_MINIO_ENDPOINT: %w", err) + } + hostParts := strings.Split(parsedURL.Hostname(), ".") + if len(hostParts) < 2 { + return nil, fmt.Errorf("unable to extract bucket from host: %s", parsedURL.Hostname()) + } + bucketName = hostParts[0] + lookup = minio.BucketLookupDNS + } else { + bucketName = "mrvabucket" + lookup = minio.BucketLookupPath + } + // TODO: unify into one. clean up state handling. + artifactstore.AF_BUCKETNAME_RESULTS = bucketName + artifactstore.AF_BUCKETNAME_PACKS = bucketName + + store, err := artifactstore.NewMinIOArtifactStore(endpoint, id, secret, lookup) if err != nil { return nil, fmt.Errorf("failed to initialize artifact store: %v", err) } - return store, nil + } func InitMinIOCodeQLDatabaseStore() (qldbstore.Store, error) { @@ -82,14 +108,38 @@ func InitMinIOCodeQLDatabaseStore() (qldbstore.Store, error) { "QLDB_MINIO_ENDPOINT", "QLDB_MINIO_ID", "QLDB_MINIO_SECRET", + "MRVA_MINIO_VIRTUAL_HOST", } validateEnvVars(requiredEnvVars) endpoint := os.Getenv("QLDB_MINIO_ENDPOINT") id := os.Getenv("QLDB_MINIO_ID") secret := os.Getenv("QLDB_MINIO_SECRET") + useVirtual := os.Getenv("MRVA_MINIO_VIRTUAL_HOST") == "1" - store, err := qldbstore.NewMinIOCodeQLDatabaseStore(endpoint, id, secret) + var lookup minio.BucketLookupType + var bucketName string + + if useVirtual { + parsedURL, err := url.Parse(endpoint) + if err != nil { + return nil, fmt.Errorf("failed to parse QLDB_MINIO_ENDPOINT: %w", err) + } + hostParts := strings.Split(parsedURL.Hostname(), ".") + if len(hostParts) < 2 { + return nil, fmt.Errorf("unable to extract bucket from host: %s", parsedURL.Hostname()) + } + bucketName = hostParts[0] + lookup = minio.BucketLookupDNS + } else { + bucketName = "mrvabucket" + lookup = minio.BucketLookupPath + } + + // TODO: unify into one. clean up state handling. + qldbstore.QL_DB_BUCKETNAME = bucketName + + store, err := qldbstore.NewMinIOCodeQLDatabaseStore(endpoint, id, secret, lookup) if err != nil { return nil, fmt.Errorf("failed to initialize ql database storage: %v", err) } diff --git a/pkg/qldbstore/qldbstore_minio.go b/pkg/qldbstore/qldbstore_minio.go index ce7c898..3274217 100644 --- a/pkg/qldbstore/qldbstore_minio.go +++ b/pkg/qldbstore/qldbstore_minio.go @@ -14,7 +14,7 @@ import ( // XX: static types: split by type? // Restrict the keys / values and centralize the common ones here -const ( +var ( QL_DB_BUCKETNAME = "mrvabucket" ) @@ -23,11 +23,13 @@ type MinIOCodeQLDatabaseStore struct { bucketName string } -func NewMinIOCodeQLDatabaseStore(endpoint, id, secret string) (*MinIOCodeQLDatabaseStore, error) { +func NewMinIOCodeQLDatabaseStore(endpoint, id, secret string, + lookup minio.BucketLookupType) (*MinIOCodeQLDatabaseStore, error) { + minioClient, err := minio.New(endpoint, &minio.Options{ Creds: credentials.NewStaticV4(id, secret, ""), Secure: false, - BucketLookup: minio.BucketLookupDNS, // Enable virtual-host-style addressing + BucketLookup: lookup, }) if err != nil { return nil, err