mrva/mrvacommander
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Code generalization: request db info from other source: remove unneccessary types

Browse Source
This commit is contained in:
Michael Hohn
2024-10-28 14:34:07 -07:00
committed by =Michael Hohn
parent 77ce997fbb
commit 52aafd6fc9
6 changed files with 56 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
pkg/qldbstore/interfaces.go
View File

@@ -4,26 +4,12 @@ import (
"mrvacommander/pkg/common"
)
type CodeQLDatabaseLocation struct {
// `Data` is a map of key-value pairs that describe the location of the database.
// For example, a simple key-value pair could be "path" -> "/path/to/database.zip".
// A more complex implementation could be "bucket" -> "example", "key" -> "unique_identifier".
// XX: static types
Data map[string]string
}
type Store interface {
// FindAvailableDBs returns a map of available databases for the requested analysisReposRequested.
// It also returns a list of repository NWOs that do not have available databases.
FindAvailableDBs(analysisReposRequested []common.NameWithOwner) (
notFoundRepos []common.NameWithOwner,
foundRepos *map[common.NameWithOwner]CodeQLDatabaseLocation)
foundRepos []common.NameWithOwner)
// GetDatabase returns the database as a byte slice for the specified repository.
// A CodeQL database is a zip archive to be processed by the CodeQL CLI.
GetDatabase(location CodeQLDatabaseLocation) ([]byte, error)
// GetDatabaseByNWO returns the database location for the specified repository.
// FindAvailableDBs should be used in lieu of this method for checking database availability.
GetDatabaseLocationByNWO(nwo common.NameWithOwner) (CodeQLDatabaseLocation, error)
// GetDatabase: return the database as a byte slice for the specified repository.
// The slice is a CodeQL database -- a zip archive to be processed by the CodeQL CLI.
GetDatabase(location common.NameWithOwner) ([]byte, error)
}

47
pkg/qldbstore/qldbstore_local.go
View File

@@ -19,48 +19,39 @@ func NewLocalFilesystemCodeQLDatabaseStore(basePath string) *FilesystemCodeQLDat
func (store *FilesystemCodeQLDatabaseStore) FindAvailableDBs(analysisReposRequested []common.NameWithOwner) (
notFoundRepos []common.NameWithOwner,
foundRepos *map[common.NameWithOwner]CodeQLDatabaseLocation) {
foundRepos []common.NameWithOwner) {
foundReposMap := make(map[common.NameWithOwner]CodeQLDatabaseLocation)
for _, repo := range analysisReposRequested {
location, err := store.GetDatabaseLocationByNWO(repo)
if err != nil {
// Form the file path
filePath := filepath.Join(store.basePath,
fmt.Sprintf("%s/%s/%s_%s_db.zip", repo.Owner, repo.Repo, repo.Owner, repo.Repo))
// Check if the file exists
if _, err := os.Stat(filePath); os.IsNotExist(err) {
notFoundRepos = append(notFoundRepos, repo)
} else {
foundReposMap[repo] = location
foundRepos = append(foundRepos, repo)
}
}
return notFoundRepos, &foundReposMap
return notFoundRepos, foundRepos
}
func (store *FilesystemCodeQLDatabaseStore) GetDatabase(location CodeQLDatabaseLocation) ([]byte, error) {
path, exists := location.Data["path"]
if !exists {
return nil, fmt.Errorf("path not specified in location")
}
func (store *FilesystemCodeQLDatabaseStore) GetDatabase(location common.NameWithOwner) ([]byte, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
return data, nil
}
func (store *FilesystemCodeQLDatabaseStore) GetDatabaseLocationByNWO(nwo common.NameWithOwner) (CodeQLDatabaseLocation, error) {
filePath := filepath.Join(store.basePath, fmt.Sprintf("%s/%s/%s_%s_db.zip", nwo.Owner, nwo.Repo, nwo.Owner, nwo.Repo))
// Form the file path
filePath := filepath.Join(store.basePath,
fmt.Sprintf("%s/%s/%s_%s_db.zip", location.Owner, location.Repo, location.Owner, location.Repo))
// Check if the file exists
if _, err := os.Stat(filePath); os.IsNotExist(err) {
return CodeQLDatabaseLocation{}, fmt.Errorf("database not found for %s", nwo)
return nil, fmt.Errorf("database not found for %s", location)
}
location := CodeQLDatabaseLocation{
Data: map[string]string{
"path": filePath,
},
// Read file and return it as byte slice
data, err := os.ReadFile(filePath)
if err != nil {
return nil, err
}
return location, nil
return data, nil
}

50
pkg/qldbstore/qldbstore_minio.go
View File

@@ -5,7 +5,6 @@ import (
"fmt"
"io"
"log/slog"
"mrvacommander/pkg/artifactstore"
"mrvacommander/pkg/common"
"github.com/minio/minio-go/v7"
@@ -49,26 +48,26 @@ func NewMinIOCodeQLDatabaseStore(endpoint, id, secret string) (*MinIOCodeQLDatab
func (store *MinIOCodeQLDatabaseStore) FindAvailableDBs(analysisReposRequested []common.NameWithOwner) (
notFoundRepos []common.NameWithOwner,
foundRepos *map[common.NameWithOwner]CodeQLDatabaseLocation) {
foundRepos []common.NameWithOwner) {
foundReposMap := make(map[common.NameWithOwner]CodeQLDatabaseLocation)
for _, repo := range analysisReposRequested {
location, err := store.GetDatabaseLocationByNWO(repo)
if err != nil {
notFoundRepos = append(notFoundRepos, repo)
status := store.haveDatabase(repo)
if status {
foundRepos = append(foundRepos, repo)
} else {
foundReposMap[repo] = location
notFoundRepos = append(notFoundRepos, repo)
}
}
return notFoundRepos, &foundReposMap
return notFoundRepos, foundRepos
}
func (store *MinIOCodeQLDatabaseStore) GetDatabase(location CodeQLDatabaseLocation) ([]byte, error) {
bucket := location.Data[artifactstore.AF_KEY_BUCKET]
key := location.Data[artifactstore.AF_KEY_KEY]
object, err := store.client.GetObject(context.Background(), bucket, key, minio.GetObjectOptions{})
func (store *MinIOCodeQLDatabaseStore) GetDatabase(location common.NameWithOwner) ([]byte, error) {
key := fmt.Sprintf("%s$%s.zip", location.Owner, location.Repo)
object, err := store.client.GetObject(context.Background(),
store.bucketName,
key,
minio.GetObjectOptions{})
if err != nil {
return nil, err
}
@@ -82,24 +81,21 @@ func (store *MinIOCodeQLDatabaseStore) GetDatabase(location CodeQLDatabaseLocati
return data, nil
}
func (store *MinIOCodeQLDatabaseStore) GetDatabaseLocationByNWO(nwo common.NameWithOwner) (CodeQLDatabaseLocation, error) {
objectName := fmt.Sprintf("%s$%s.zip", nwo.Owner, nwo.Repo)
func (store *MinIOCodeQLDatabaseStore) haveDatabase(location common.NameWithOwner) bool {
objectName := fmt.Sprintf("%s$%s.zip", location.Owner, location.Repo)
// Check if the object exists
_, err := store.client.StatObject(context.Background(), store.bucketName, objectName, minio.StatObjectOptions{})
_, err := store.client.StatObject(context.Background(),
store.bucketName,
objectName,
minio.StatObjectOptions{})
if err != nil {
if minio.ToErrorResponse(err).Code == "NoSuchKey" {
return CodeQLDatabaseLocation{}, fmt.Errorf("database not found for %s", nwo)
slog.Info("No database found for", location)
return false
}
return CodeQLDatabaseLocation{}, err
slog.Info("General database error while checking for", location)
return false
}
location := CodeQLDatabaseLocation{
Data: map[string]string{
artifactstore.AF_KEY_BUCKET: store.bucketName,
artifactstore.AF_KEY_KEY: objectName,
},
}
return location, nil
return true
}