mrva/mrvacommander
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Code generalization: request db info from other source: remove unneccessary types

Browse Source
This commit is contained in:
Michael Hohn
2024-10-28 14:34:07 -07:00
committed by =Michael Hohn
parent 77ce997fbb
commit 52aafd6fc9
6 changed files with 56 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
pkg/agent/agent.go
View File

@@ -184,13 +184,7 @@ func RunAnalysisJob(
return result, fmt.Errorf("failed to extract query pack: %w", err)
}
// Download the CodeQL database as a byte slice
location, err := dbs.GetDatabaseLocationByNWO(job.Spec.NameWithOwner)
if err != nil {
return result, fmt.Errorf("failed to get database location: %w", err)
}
databaseData, err := dbs.GetDatabase(location)
databaseData, err := dbs.GetDatabase(job.Spec.NameWithOwner)
if err != nil {
return result, fmt.Errorf("failed to get database: %w", err)
}

16
pkg/codeql/codeql.go
View File

@@ -526,25 +526,9 @@ func generateSarif(codeql CodeqlCli, language queue.QueryLanguage, databasePath,
return nil, err
}
slog.Debug("XX: sarif original", "sarif=", sarifData)
slog.Debug("XX: sarif modified", "modifiedJSON=", modifiedJSON)
return modifiedJSON, nil
}
// XX: inlined this function
// func injectVersionControlInfo(sarif *Sarif, nwo, databaseSHA string) {
// // XX: is nwo name/owner or language?
// slog.Debug("XX: 2: is nwo a name/owner, or the original callers' queryLanguage?",
// "nwo", nwo)
// for _, run := range sarif.Runs {
// run.VersionControlProvenance = append(run.VersionControlProvenance, map[string]interface{}{
// "repositoryUri": fmt.Sprintf("%s/%s", os.Getenv("GITHUB_SERVER_URL"), nwo),
// "revisionId": databaseSHA,
// })
// }
// }
// getSarifResultCount returns the number of results in the SARIF file.
func getSarifResultCount(sarif []byte) int {
var sarifData Sarif

22
pkg/qldbstore/interfaces.go
View File

@@ -4,26 +4,12 @@ import (
"mrvacommander/pkg/common"
)
type CodeQLDatabaseLocation struct {
// `Data` is a map of key-value pairs that describe the location of the database.
// For example, a simple key-value pair could be "path" -> "/path/to/database.zip".
// A more complex implementation could be "bucket" -> "example", "key" -> "unique_identifier".
// XX: static types
Data map[string]string
}
type Store interface {
// FindAvailableDBs returns a map of available databases for the requested analysisReposRequested.
// It also returns a list of repository NWOs that do not have available databases.
FindAvailableDBs(analysisReposRequested []common.NameWithOwner) (
notFoundRepos []common.NameWithOwner,
foundRepos *map[common.NameWithOwner]CodeQLDatabaseLocation)
foundRepos []common.NameWithOwner)
// GetDatabase returns the database as a byte slice for the specified repository.
// A CodeQL database is a zip archive to be processed by the CodeQL CLI.
GetDatabase(location CodeQLDatabaseLocation) ([]byte, error)
// GetDatabaseByNWO returns the database location for the specified repository.
// FindAvailableDBs should be used in lieu of this method for checking database availability.
GetDatabaseLocationByNWO(nwo common.NameWithOwner) (CodeQLDatabaseLocation, error)
// GetDatabase: return the database as a byte slice for the specified repository.
// The slice is a CodeQL database -- a zip archive to be processed by the CodeQL CLI.
GetDatabase(location common.NameWithOwner) ([]byte, error)
}

65
pkg/qldbstore/qldbstore_local.go
View File

@@ -19,48 +19,39 @@ func NewLocalFilesystemCodeQLDatabaseStore(basePath string) *FilesystemCodeQLDat
func (store *FilesystemCodeQLDatabaseStore) FindAvailableDBs(analysisReposRequested []common.NameWithOwner) (
notFoundRepos []common.NameWithOwner,
foundRepos *map[common.NameWithOwner]CodeQLDatabaseLocation) {
foundRepos []common.NameWithOwner) {
foundReposMap := make(map[common.NameWithOwner]CodeQLDatabaseLocation)
for _, repo := range analysisReposRequested {
location, err := store.GetDatabaseLocationByNWO(repo)
if err != nil {
notFoundRepos = append(notFoundRepos, repo)
} else {
foundReposMap[repo] = location
}
}
return notFoundRepos, &foundReposMap
}
func (store *FilesystemCodeQLDatabaseStore) GetDatabase(location CodeQLDatabaseLocation) ([]byte, error) {
path, exists := location.Data["path"]
if !exists {
return nil, fmt.Errorf("path not specified in location")
}
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
return data, nil
}
func (store *FilesystemCodeQLDatabaseStore) GetDatabaseLocationByNWO(nwo common.NameWithOwner) (CodeQLDatabaseLocation, error) {
filePath := filepath.Join(store.basePath, fmt.Sprintf("%s/%s/%s_%s_db.zip", nwo.Owner, nwo.Repo, nwo.Owner, nwo.Repo))
// Form the file path
filePath := filepath.Join(store.basePath,
fmt.Sprintf("%s/%s/%s_%s_db.zip", repo.Owner, repo.Repo, repo.Owner, repo.Repo))
// Check if the file exists
if _, err := os.Stat(filePath); os.IsNotExist(err) {
return CodeQLDatabaseLocation{}, fmt.Errorf("database not found for %s", nwo)
notFoundRepos = append(notFoundRepos, repo)
} else {
foundRepos = append(foundRepos, repo)
}
}
location := CodeQLDatabaseLocation{
Data: map[string]string{
"path": filePath,
},
}
return location, nil
return notFoundRepos, foundRepos
}
func (store *FilesystemCodeQLDatabaseStore) GetDatabase(location common.NameWithOwner) ([]byte, error) {
// Form the file path
filePath := filepath.Join(store.basePath,
fmt.Sprintf("%s/%s/%s_%s_db.zip", location.Owner, location.Repo, location.Owner, location.Repo))
// Check if the file exists
if _, err := os.Stat(filePath); os.IsNotExist(err) {
return nil, fmt.Errorf("database not found for %s", location)
}
// Read file and return it as byte slice
data, err := os.ReadFile(filePath)
if err != nil {
return nil, err
}
return data, nil
}

50
pkg/qldbstore/qldbstore_minio.go
View File

@@ -5,7 +5,6 @@ import (
"fmt"
"io"
"log/slog"
"mrvacommander/pkg/artifactstore"
"mrvacommander/pkg/common"
"github.com/minio/minio-go/v7"
@@ -49,26 +48,26 @@ func NewMinIOCodeQLDatabaseStore(endpoint, id, secret string) (*MinIOCodeQLDatab
func (store *MinIOCodeQLDatabaseStore) FindAvailableDBs(analysisReposRequested []common.NameWithOwner) (
notFoundRepos []common.NameWithOwner,
foundRepos *map[common.NameWithOwner]CodeQLDatabaseLocation) {
foundRepos []common.NameWithOwner) {
foundReposMap := make(map[common.NameWithOwner]CodeQLDatabaseLocation)
for _, repo := range analysisReposRequested {
location, err := store.GetDatabaseLocationByNWO(repo)
if err != nil {
notFoundRepos = append(notFoundRepos, repo)
status := store.haveDatabase(repo)
if status {
foundRepos = append(foundRepos, repo)
} else {
foundReposMap[repo] = location
notFoundRepos = append(notFoundRepos, repo)
}
}
return notFoundRepos, &foundReposMap
return notFoundRepos, foundRepos
}
func (store *MinIOCodeQLDatabaseStore) GetDatabase(location CodeQLDatabaseLocation) ([]byte, error) {
bucket := location.Data[artifactstore.AF_KEY_BUCKET]
key := location.Data[artifactstore.AF_KEY_KEY]
object, err := store.client.GetObject(context.Background(), bucket, key, minio.GetObjectOptions{})
func (store *MinIOCodeQLDatabaseStore) GetDatabase(location common.NameWithOwner) ([]byte, error) {
key := fmt.Sprintf("%s$%s.zip", location.Owner, location.Repo)
object, err := store.client.GetObject(context.Background(),
store.bucketName,
key,
minio.GetObjectOptions{})
if err != nil {
return nil, err
}
@@ -82,24 +81,21 @@ func (store *MinIOCodeQLDatabaseStore) GetDatabase(location CodeQLDatabaseLocati
return data, nil
}
func (store *MinIOCodeQLDatabaseStore) GetDatabaseLocationByNWO(nwo common.NameWithOwner) (CodeQLDatabaseLocation, error) {
objectName := fmt.Sprintf("%s$%s.zip", nwo.Owner, nwo.Repo)
func (store *MinIOCodeQLDatabaseStore) haveDatabase(location common.NameWithOwner) bool {
objectName := fmt.Sprintf("%s$%s.zip", location.Owner, location.Repo)
// Check if the object exists
_, err := store.client.StatObject(context.Background(), store.bucketName, objectName, minio.StatObjectOptions{})
_, err := store.client.StatObject(context.Background(),
store.bucketName,
objectName,
minio.StatObjectOptions{})
if err != nil {
if minio.ToErrorResponse(err).Code == "NoSuchKey" {
return CodeQLDatabaseLocation{}, fmt.Errorf("database not found for %s", nwo)
slog.Info("No database found for", location)
return false
}
return CodeQLDatabaseLocation{}, err
slog.Info("General database error while checking for", location)
return false
}
location := CodeQLDatabaseLocation{
Data: map[string]string{
artifactstore.AF_KEY_BUCKET: store.bucketName,
artifactstore.AF_KEY_KEY: objectName,
},
}
return location, nil
return true
}

24
pkg/server/server.go
View File

@@ -16,7 +16,6 @@ import (
"mrvacommander/pkg/artifactstore"
"mrvacommander/pkg/common"
"mrvacommander/pkg/qldbstore"
"mrvacommander/pkg/queue"
"mrvacommander/utils"
@@ -24,14 +23,14 @@ import (
)
func (c *CommanderSingle) startAnalyses(
analysisRepos *map[common.NameWithOwner]qldbstore.CodeQLDatabaseLocation,
analysisRepos []common.NameWithOwner,
queryPackLocation artifactstore.ArtifactLocation,
sessionId int,
queryLanguage queue.QueryLanguage) {
slog.Debug("Queueing analysis jobs", "count", len(*analysisRepos))
slog.Debug("Queueing analysis jobs", "count", len(analysisRepos))
for nwo := range *analysisRepos {
for _, nwo := range analysisRepos {
jobSpec := common.JobSpec{
SessionID: sessionId,
NameWithOwner: nwo,
@@ -380,25 +379,20 @@ func (c *CommanderSingle) MRVADownloadQLDB(w http.ResponseWriter, r *http.Reques
// This is a direct data request -- don't reply with a download url.
vars := mux.Vars(r)
owner := vars["repo_owner"]
name := vars["repo_name"]
dbl := qldbstore.CodeQLDatabaseLocation{
Data: map[string]string{
qldbstore.QL_KEY_BUCKET: qldbstore.QL_DB_BUCKETNAME,
qldbstore.QL_KEY_KEY: fmt.Sprintf("%s$%s.zip", owner, name),
},
dbl := common.NameWithOwner{
Owner: vars["repo_owner"],
Repo: vars["repo_name"],
}
slog.Debug("Returning codeql database using database location",
"qldbstore.CodeQLDatabaseLocation", dbl,
"dbl", dbl,
)
dbContent, err := c.v.CodeQLDBStore.GetDatabase(dbl)
if err != nil {
slog.Error("Failed to retrieve ql database",
"error", err,
"qldbstore.CodeQLDatabaseLocation", dbl,
"dbl", dbl,
)
http.Error(w, "Failed to retrieve ql database", http.StatusInternalServerError)
return
@@ -568,7 +562,7 @@ func (c *CommanderSingle) MRVARequestCommon(w http.ResponseWriter, r *http.Reque
notFoundRepos, analysisRepos := c.v.CodeQLDBStore.FindAvailableDBs(repoNWOs)
if len(*analysisRepos) == 0 {
if len(analysisRepos) == 0 {
slog.Warn("No repositories found for analysis")
}