From 0a52b729cd4a6cd918f32f3fdeafd14fa6f3f1b2 Mon Sep 17 00:00:00 2001 From: Michael Hohn Date: Wed, 14 Aug 2024 13:01:15 -0700 Subject: [PATCH] Expand the codeql db download response End-to-end testing contained an unhandled CodeQL database download request. The handlers are added in this patch. Debugging info is below for reference. The mrvacommander *server* fails with the following. The source code is : func setupEndpoints(c CommanderAPI) See mrvacommander/pkg/server/server.go, endpoints for getting a URL to download artifacts. Original Downloading artifacts for tdlib_telegram-bot-apictsj8529d9_2 ... Downloading database tdlib/telegram-bot-apictsj8529d9 cpp mirva-session-1400 tdlib_telegram-bot-apictsj8529d9_2 ... 2024/08/13 12:31:38 >> GET http://localhost:8080/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp ... 2024/08/13 12:31:38 << 404 http://localhost:8080/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp ... -rwxr-xr-x@ 1 hohn staff 169488 Aug 13 12:29 tdlib_telegram-bot-apictsj8529d9_2.sarif* -rwxr-xr-x@ 1 hohn staff 10 Aug 13 12:31 tdlib_telegram-bot-apictsj8529d9_2_db.zip* Server log server | 2024/08/13 19:31:38 ERROR Unhandled endpoint method=GET uri=/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp Try a manual download from the server 8:$ wget http://localhost:8080/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp --2024-08-13 12:56:05-- http://localhost:8080/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp Resolving localhost (localhost)... ::1, 127.0.0.1 Connecting to localhost (localhost)|::1|:8080... connected. HTTP request sent, awaiting response... 404 Not Found 2024-08-13 12:56:05 ERROR 404: Not Found. server | 2024/08/13 19:56:05 ERROR Unhandled endpoint method=GET uri=/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp The full info for the DB tdlib,telegram-bot-api,8529d9,2.17.0,2024-05-09 08:02:49.545174+00:00,cpp,f95d406da67adb8ac13d9c562291aa57c65398e0,306106.0,/Users/hohn/work-gh/mrva/mrva-open-source-download/repos-2024-04-29/tdlib/telegram-bot-api/code-scanning/codeql/databases/cpp/db.zip,cpp,C/C++,1244.0,306106.0,2024-05-13T15:54:54.749093,cpp,True,3375,373477635 The gh-mrva *client* sends the following. The source is gh-mrva/utils/utils.go, client.Get(fmt.Sprintf("http://localhost:8080/repos/%s/code-scanning/codeql/databases/%s", task.Nwo, task.Language)) We have cd /Users/hohn/work-gh/mrva/gh-mrva 0:$ rg 'repos/.*/code-scanning/codeql/databases' ... utils/utils.go 625: // resp, err := client.Get(fmt.Sprintf("https://api.github.com/repos/%s/code-scanning/codeql/databases/%s", task.Nwo, task.Language)) 626: resp, err := client.Get(fmt.Sprintf("http://localhost:8080/repos/%s/code-scanning/codeql/databases/%s", task.Nwo, task.Language)) And resp, err := client.Get(fmt.Sprintf("http://localhost:8080/repos/%s/code-scanning/codeql/databases/%s", task.Nwo, task.Language)) The original DB upload was cd ~/work-gh/mrva/mrvacommander/client/qldbtools && \ ./bin/mc-db-populate-minio -n 11 < scratch/db-info-3.csv ... 2024-08-14 09:29:19 [INFO] Uploaded /Users/hohn/work-gh/mrva/mrva-open-source-download/repos-2024-04-29/tdlib/telegram-bot-api/code-scanning/codeql/databases/cpp/db.zip as tdlib$telegram-bot-apictsj8529d9.zip to bucket qldb ... --- pkg/qldbstore/interfaces.go | 4 +-- pkg/qldbstore/qldbstore_local.go | 4 +-- pkg/qldbstore/qldbstore_minio.go | 14 +++++++--- pkg/server/interfaces.go | 1 + pkg/server/server.go | 47 ++++++++++++++++++++++++++++++++ 5 files changed, 62 insertions(+), 8 deletions(-) diff --git a/pkg/qldbstore/interfaces.go b/pkg/qldbstore/interfaces.go index 04bbd6a..6c7f87e 100644 --- a/pkg/qldbstore/interfaces.go +++ b/pkg/qldbstore/interfaces.go @@ -5,11 +5,11 @@ import ( ) type CodeQLDatabaseLocation struct { - // `data` is a map of key-value pairs that describe the location of the database. + // `Data` is a map of key-value pairs that describe the location of the database. // For example, a simple key-value pair could be "path" -> "/path/to/database.zip". // A more complex implementation could be "bucket" -> "example", "key" -> "unique_identifier". // XX: static types - data map[string]string + Data map[string]string } type Store interface { diff --git a/pkg/qldbstore/qldbstore_local.go b/pkg/qldbstore/qldbstore_local.go index ec24bb3..1903cf6 100644 --- a/pkg/qldbstore/qldbstore_local.go +++ b/pkg/qldbstore/qldbstore_local.go @@ -35,7 +35,7 @@ func (store *FilesystemCodeQLDatabaseStore) FindAvailableDBs(analysisReposReques } func (store *FilesystemCodeQLDatabaseStore) GetDatabase(location CodeQLDatabaseLocation) ([]byte, error) { - path, exists := location.data["path"] + path, exists := location.Data["path"] if !exists { return nil, fmt.Errorf("path not specified in location") } @@ -57,7 +57,7 @@ func (store *FilesystemCodeQLDatabaseStore) GetDatabaseLocationByNWO(nwo common. } location := CodeQLDatabaseLocation{ - data: map[string]string{ + Data: map[string]string{ "path": filePath, }, } diff --git a/pkg/qldbstore/qldbstore_minio.go b/pkg/qldbstore/qldbstore_minio.go index d91e7db..8670957 100644 --- a/pkg/qldbstore/qldbstore_minio.go +++ b/pkg/qldbstore/qldbstore_minio.go @@ -12,7 +12,13 @@ import ( "github.com/minio/minio-go/v7/pkg/credentials" ) -const QL_DB_BUCKETNAME = "qldb" +// XX: static types: split by type? +// Restrict the keys / values and centralize the common ones here +const ( + QL_DB_BUCKETNAME = "qldb" + QL_KEY_BUCKET = "bucket" + QL_KEY_KEY = "key" +) type MinIOCodeQLDatabaseStore struct { client *minio.Client @@ -59,8 +65,8 @@ func (store *MinIOCodeQLDatabaseStore) FindAvailableDBs(analysisReposRequested [ } func (store *MinIOCodeQLDatabaseStore) GetDatabase(location CodeQLDatabaseLocation) ([]byte, error) { - bucket := location.data[artifactstore.AF_KEY_BUCKET] - key := location.data[artifactstore.AF_KEY_KEY] + bucket := location.Data[artifactstore.AF_KEY_BUCKET] + key := location.Data[artifactstore.AF_KEY_KEY] object, err := store.client.GetObject(context.Background(), bucket, key, minio.GetObjectOptions{}) if err != nil { @@ -89,7 +95,7 @@ func (store *MinIOCodeQLDatabaseStore) GetDatabaseLocationByNWO(nwo common.NameW } location := CodeQLDatabaseLocation{ - data: map[string]string{ + Data: map[string]string{ artifactstore.AF_KEY_BUCKET: store.bucketName, artifactstore.AF_KEY_KEY: objectName, }, diff --git a/pkg/server/interfaces.go b/pkg/server/interfaces.go index 2438f11..b25c190 100644 --- a/pkg/server/interfaces.go +++ b/pkg/server/interfaces.go @@ -10,5 +10,6 @@ type CommanderAPI interface { MRVAStatus(w http.ResponseWriter, r *http.Request) MRVADownloadArtifactID(w http.ResponseWriter, r *http.Request) MRVADownloadArtifact(w http.ResponseWriter, r *http.Request) + MRVADownloadQLDB(w http.ResponseWriter, r *http.Request) MRVADownloadServe(w http.ResponseWriter, r *http.Request) } diff --git a/pkg/server/server.go b/pkg/server/server.go index 886f02c..0b6924c 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -63,7 +63,11 @@ func setupEndpoints(c CommanderAPI) { r.HandleFunc("/repos/{owner}/{repo}/code-scanning/codeql/variant-analyses/{codeql_variant_analysis_id}", c.MRVAStatus) r.HandleFunc("/repositories/{controller_repo_id}/code-scanning/codeql/variant-analyses/{codeql_variant_analysis_id}", c.MRVAStatusID) + // XX: Handle endpoint + // /repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp // Endpoints for getting a URL to download artifacts + // /repos/tdlib /telegram.../code-scanning/codeql/databases/cpp + r.HandleFunc("/repos/{repo_owner}/{repo_name}/code-scanning/codeql/databases/{repo_language}", c.MRVADownloadQLDB) r.HandleFunc("/repos/{controller_owner}/{controller_repo}/code-scanning/codeql/variant-analyses/{codeql_variant_analysis_id}/repos/{repo_owner}/{repo_name}", c.MRVADownloadArtifact) r.HandleFunc("/repositories/{controller_repo_id}/code-scanning/codeql/variant-analyses/{codeql_variant_analysis_id}/repositories/{repository_id}", c.MRVADownloadArtifactID) @@ -362,6 +366,49 @@ func (c *CommanderSingle) MRVADownloadArtifactID(w http.ResponseWriter, r *http. c.MRVADownloadArtifactCommon(w, r, int(repoId), jobSpec) } +func (c *CommanderSingle) MRVADownloadQLDB(w http.ResponseWriter, r *http.Request) { + // The repositories are uploaded without language and can be downloaded + // without it. We ignore the language parameter passed in the request: + // vars["repo_language"] + + // Other artifact downloads, like sendArtifactDownloadResponse, depend on + // a jobspec (integer job id). This request has none, and needs none. + + // An original upload example is + // tdlib$telegram-bot-apictsj8529d9.zip to bucket qldb. + + // This is a direct data request -- don't reply with a download url. + + vars := mux.Vars(r) + owner := vars["repo_owner"] + name := vars["repo_name"] + + dbl := qldbstore.CodeQLDatabaseLocation{ + Data: map[string]string{ + qldbstore.QL_KEY_BUCKET: qldbstore.QL_DB_BUCKETNAME, + qldbstore.QL_KEY_KEY: fmt.Sprintf("%s$%s.zip", owner, name), + }, + } + + slog.Debug("Returning codeql database using database location", + "qldbstore.CodeQLDatabaseLocation", dbl, + ) + + dbContent, err := c.v.CodeQLDBStore.GetDatabase(dbl) + if err != nil { + slog.Error("Failed to retrieve ql database", + "error", err, + "qldbstore.CodeQLDatabaseLocation", dbl, + ) + http.Error(w, "Failed to retrieve ql database", http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/octet-stream") + w.Write(dbContent) + +} + func (c *CommanderSingle) MRVADownloadArtifact(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r)