diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 38ef41a..ac2a83e 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -184,13 +184,7 @@ func RunAnalysisJob( return result, fmt.Errorf("failed to extract query pack: %w", err) } - // Download the CodeQL database as a byte slice - location, err := dbs.GetDatabaseLocationByNWO(job.Spec.NameWithOwner) - if err != nil { - return result, fmt.Errorf("failed to get database location: %w", err) - } - - databaseData, err := dbs.GetDatabase(location) + databaseData, err := dbs.GetDatabase(job.Spec.NameWithOwner) if err != nil { return result, fmt.Errorf("failed to get database: %w", err) } diff --git a/pkg/codeql/codeql.go b/pkg/codeql/codeql.go index d521c6e..3dc9c35 100644 --- a/pkg/codeql/codeql.go +++ b/pkg/codeql/codeql.go @@ -526,25 +526,9 @@ func generateSarif(codeql CodeqlCli, language queue.QueryLanguage, databasePath, return nil, err } - slog.Debug("XX: sarif original", "sarif=", sarifData) - slog.Debug("XX: sarif modified", "modifiedJSON=", modifiedJSON) - return modifiedJSON, nil } -// XX: inlined this function -// func injectVersionControlInfo(sarif *Sarif, nwo, databaseSHA string) { -// // XX: is nwo name/owner or language? -// slog.Debug("XX: 2: is nwo a name/owner, or the original callers' queryLanguage?", -// "nwo", nwo) -// for _, run := range sarif.Runs { -// run.VersionControlProvenance = append(run.VersionControlProvenance, map[string]interface{}{ -// "repositoryUri": fmt.Sprintf("%s/%s", os.Getenv("GITHUB_SERVER_URL"), nwo), -// "revisionId": databaseSHA, -// }) -// } -// } - // getSarifResultCount returns the number of results in the SARIF file. func getSarifResultCount(sarif []byte) int { var sarifData Sarif diff --git a/pkg/qldbstore/interfaces.go b/pkg/qldbstore/interfaces.go index 6c7f87e..db239d3 100644 --- a/pkg/qldbstore/interfaces.go +++ b/pkg/qldbstore/interfaces.go @@ -4,26 +4,12 @@ import ( "mrvacommander/pkg/common" ) -type CodeQLDatabaseLocation struct { - // `Data` is a map of key-value pairs that describe the location of the database. - // For example, a simple key-value pair could be "path" -> "/path/to/database.zip". - // A more complex implementation could be "bucket" -> "example", "key" -> "unique_identifier". - // XX: static types - Data map[string]string -} - type Store interface { - // FindAvailableDBs returns a map of available databases for the requested analysisReposRequested. - // It also returns a list of repository NWOs that do not have available databases. FindAvailableDBs(analysisReposRequested []common.NameWithOwner) ( notFoundRepos []common.NameWithOwner, - foundRepos *map[common.NameWithOwner]CodeQLDatabaseLocation) + foundRepos []common.NameWithOwner) - // GetDatabase returns the database as a byte slice for the specified repository. - // A CodeQL database is a zip archive to be processed by the CodeQL CLI. - GetDatabase(location CodeQLDatabaseLocation) ([]byte, error) - - // GetDatabaseByNWO returns the database location for the specified repository. - // FindAvailableDBs should be used in lieu of this method for checking database availability. - GetDatabaseLocationByNWO(nwo common.NameWithOwner) (CodeQLDatabaseLocation, error) + // GetDatabase: return the database as a byte slice for the specified repository. + // The slice is a CodeQL database -- a zip archive to be processed by the CodeQL CLI. + GetDatabase(location common.NameWithOwner) ([]byte, error) } diff --git a/pkg/qldbstore/qldbstore_local.go b/pkg/qldbstore/qldbstore_local.go index 1903cf6..4368de8 100644 --- a/pkg/qldbstore/qldbstore_local.go +++ b/pkg/qldbstore/qldbstore_local.go @@ -19,48 +19,39 @@ func NewLocalFilesystemCodeQLDatabaseStore(basePath string) *FilesystemCodeQLDat func (store *FilesystemCodeQLDatabaseStore) FindAvailableDBs(analysisReposRequested []common.NameWithOwner) ( notFoundRepos []common.NameWithOwner, - foundRepos *map[common.NameWithOwner]CodeQLDatabaseLocation) { + foundRepos []common.NameWithOwner) { - foundReposMap := make(map[common.NameWithOwner]CodeQLDatabaseLocation) for _, repo := range analysisReposRequested { - location, err := store.GetDatabaseLocationByNWO(repo) - if err != nil { + // Form the file path + filePath := filepath.Join(store.basePath, + fmt.Sprintf("%s/%s/%s_%s_db.zip", repo.Owner, repo.Repo, repo.Owner, repo.Repo)) + + // Check if the file exists + if _, err := os.Stat(filePath); os.IsNotExist(err) { notFoundRepos = append(notFoundRepos, repo) } else { - foundReposMap[repo] = location + foundRepos = append(foundRepos, repo) } } - return notFoundRepos, &foundReposMap + return notFoundRepos, foundRepos } -func (store *FilesystemCodeQLDatabaseStore) GetDatabase(location CodeQLDatabaseLocation) ([]byte, error) { - path, exists := location.Data["path"] - if !exists { - return nil, fmt.Errorf("path not specified in location") - } +func (store *FilesystemCodeQLDatabaseStore) GetDatabase(location common.NameWithOwner) ([]byte, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - - return data, nil -} - -func (store *FilesystemCodeQLDatabaseStore) GetDatabaseLocationByNWO(nwo common.NameWithOwner) (CodeQLDatabaseLocation, error) { - filePath := filepath.Join(store.basePath, fmt.Sprintf("%s/%s/%s_%s_db.zip", nwo.Owner, nwo.Repo, nwo.Owner, nwo.Repo)) + // Form the file path + filePath := filepath.Join(store.basePath, + fmt.Sprintf("%s/%s/%s_%s_db.zip", location.Owner, location.Repo, location.Owner, location.Repo)) // Check if the file exists if _, err := os.Stat(filePath); os.IsNotExist(err) { - return CodeQLDatabaseLocation{}, fmt.Errorf("database not found for %s", nwo) + return nil, fmt.Errorf("database not found for %s", location) } - location := CodeQLDatabaseLocation{ - Data: map[string]string{ - "path": filePath, - }, + // Read file and return it as byte slice + data, err := os.ReadFile(filePath) + if err != nil { + return nil, err } - - return location, nil + return data, nil } diff --git a/pkg/qldbstore/qldbstore_minio.go b/pkg/qldbstore/qldbstore_minio.go index 8670957..7c426bf 100644 --- a/pkg/qldbstore/qldbstore_minio.go +++ b/pkg/qldbstore/qldbstore_minio.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "log/slog" - "mrvacommander/pkg/artifactstore" "mrvacommander/pkg/common" "github.com/minio/minio-go/v7" @@ -49,26 +48,26 @@ func NewMinIOCodeQLDatabaseStore(endpoint, id, secret string) (*MinIOCodeQLDatab func (store *MinIOCodeQLDatabaseStore) FindAvailableDBs(analysisReposRequested []common.NameWithOwner) ( notFoundRepos []common.NameWithOwner, - foundRepos *map[common.NameWithOwner]CodeQLDatabaseLocation) { + foundRepos []common.NameWithOwner) { - foundReposMap := make(map[common.NameWithOwner]CodeQLDatabaseLocation) for _, repo := range analysisReposRequested { - location, err := store.GetDatabaseLocationByNWO(repo) - if err != nil { - notFoundRepos = append(notFoundRepos, repo) + status := store.haveDatabase(repo) + if status { + foundRepos = append(foundRepos, repo) } else { - foundReposMap[repo] = location + notFoundRepos = append(notFoundRepos, repo) } } - return notFoundRepos, &foundReposMap + return notFoundRepos, foundRepos } -func (store *MinIOCodeQLDatabaseStore) GetDatabase(location CodeQLDatabaseLocation) ([]byte, error) { - bucket := location.Data[artifactstore.AF_KEY_BUCKET] - key := location.Data[artifactstore.AF_KEY_KEY] - - object, err := store.client.GetObject(context.Background(), bucket, key, minio.GetObjectOptions{}) +func (store *MinIOCodeQLDatabaseStore) GetDatabase(location common.NameWithOwner) ([]byte, error) { + key := fmt.Sprintf("%s$%s.zip", location.Owner, location.Repo) + object, err := store.client.GetObject(context.Background(), + store.bucketName, + key, + minio.GetObjectOptions{}) if err != nil { return nil, err } @@ -82,24 +81,21 @@ func (store *MinIOCodeQLDatabaseStore) GetDatabase(location CodeQLDatabaseLocati return data, nil } -func (store *MinIOCodeQLDatabaseStore) GetDatabaseLocationByNWO(nwo common.NameWithOwner) (CodeQLDatabaseLocation, error) { - objectName := fmt.Sprintf("%s$%s.zip", nwo.Owner, nwo.Repo) +func (store *MinIOCodeQLDatabaseStore) haveDatabase(location common.NameWithOwner) bool { + objectName := fmt.Sprintf("%s$%s.zip", location.Owner, location.Repo) // Check if the object exists - _, err := store.client.StatObject(context.Background(), store.bucketName, objectName, minio.StatObjectOptions{}) + _, err := store.client.StatObject(context.Background(), + store.bucketName, + objectName, + minio.StatObjectOptions{}) if err != nil { if minio.ToErrorResponse(err).Code == "NoSuchKey" { - return CodeQLDatabaseLocation{}, fmt.Errorf("database not found for %s", nwo) + slog.Info("No database found for", location) + return false } - return CodeQLDatabaseLocation{}, err + slog.Info("General database error while checking for", location) + return false } - - location := CodeQLDatabaseLocation{ - Data: map[string]string{ - artifactstore.AF_KEY_BUCKET: store.bucketName, - artifactstore.AF_KEY_KEY: objectName, - }, - } - - return location, nil + return true } diff --git a/pkg/server/server.go b/pkg/server/server.go index 0b6924c..25a49c8 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -16,7 +16,6 @@ import ( "mrvacommander/pkg/artifactstore" "mrvacommander/pkg/common" - "mrvacommander/pkg/qldbstore" "mrvacommander/pkg/queue" "mrvacommander/utils" @@ -24,14 +23,14 @@ import ( ) func (c *CommanderSingle) startAnalyses( - analysisRepos *map[common.NameWithOwner]qldbstore.CodeQLDatabaseLocation, + analysisRepos []common.NameWithOwner, queryPackLocation artifactstore.ArtifactLocation, sessionId int, queryLanguage queue.QueryLanguage) { - slog.Debug("Queueing analysis jobs", "count", len(*analysisRepos)) + slog.Debug("Queueing analysis jobs", "count", len(analysisRepos)) - for nwo := range *analysisRepos { + for _, nwo := range analysisRepos { jobSpec := common.JobSpec{ SessionID: sessionId, NameWithOwner: nwo, @@ -380,25 +379,20 @@ func (c *CommanderSingle) MRVADownloadQLDB(w http.ResponseWriter, r *http.Reques // This is a direct data request -- don't reply with a download url. vars := mux.Vars(r) - owner := vars["repo_owner"] - name := vars["repo_name"] - - dbl := qldbstore.CodeQLDatabaseLocation{ - Data: map[string]string{ - qldbstore.QL_KEY_BUCKET: qldbstore.QL_DB_BUCKETNAME, - qldbstore.QL_KEY_KEY: fmt.Sprintf("%s$%s.zip", owner, name), - }, + dbl := common.NameWithOwner{ + Owner: vars["repo_owner"], + Repo: vars["repo_name"], } slog.Debug("Returning codeql database using database location", - "qldbstore.CodeQLDatabaseLocation", dbl, + "dbl", dbl, ) dbContent, err := c.v.CodeQLDBStore.GetDatabase(dbl) if err != nil { slog.Error("Failed to retrieve ql database", "error", err, - "qldbstore.CodeQLDatabaseLocation", dbl, + "dbl", dbl, ) http.Error(w, "Failed to retrieve ql database", http.StatusInternalServerError) return @@ -568,7 +562,7 @@ func (c *CommanderSingle) MRVARequestCommon(w http.ResponseWriter, r *http.Reque notFoundRepos, analysisRepos := c.v.CodeQLDBStore.FindAvailableDBs(repoNWOs) - if len(*analysisRepos) == 0 { + if len(analysisRepos) == 0 { slog.Warn("No repositories found for analysis") }