mrva/mrvacommander
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Expand the codeql db download response

Browse Source 
End-to-end testing contained an unhandled CodeQL database download
request.  The handlers are added in this patch.  Debugging info is below
for reference.

The mrvacommander *server* fails with the following.  The source code is
: func setupEndpoints(c CommanderAPI)
See mrvacommander/pkg/server/server.go, endpoints for getting a URL to download artifacts.

  Original

          Downloading artifacts for tdlib_telegram-bot-apictsj8529d9_2
          ...
          Downloading database tdlib/telegram-bot-apictsj8529d9 cpp mirva-session-1400 tdlib_telegram-bot-apictsj8529d9_2
          ...
          2024/08/13 12:31:38 >> GET http://localhost:8080/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp
          ...
          2024/08/13 12:31:38 << 404 http://localhost:8080/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp
          ...
          -rwxr-xr-x@  1 hohn  staff  169488 Aug 13 12:29 tdlib_telegram-bot-apictsj8529d9_2.sarif*
          -rwxr-xr-x@  1 hohn  staff      10 Aug 13 12:31 tdlib_telegram-bot-apictsj8529d9_2_db.zip*

  Server log

          server         | 2024/08/13 19:31:38 ERROR Unhandled endpoint method=GET uri=/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp

  Try a manual download from the server

          8:$ wget http://localhost:8080/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp
          --2024-08-13 12:56:05--  http://localhost:8080/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp
          Resolving localhost (localhost)... ::1, 127.0.0.1
          Connecting to localhost (localhost)|::1|:8080... connected.
          HTTP request sent, awaiting response... 404 Not Found
          2024-08-13 12:56:05 ERROR 404: Not Found.

          server         | 2024/08/13 19:56:05 ERROR Unhandled endpoint method=GET uri=/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp

  The full info for the DB
          tdlib,telegram-bot-api,8529d9,2.17.0,2024-05-09 08:02:49.545174+00:00,cpp,f95d406da67adb8ac13d9c562291aa57c65398e0,306106.0,/Users/hohn/work-gh/mrva/mrva-open-source-download/repos-2024-04-29/tdlib/telegram-bot-api/code-scanning/codeql/databases/cpp/db.zip,cpp,C/C++,1244.0,306106.0,2024-05-13T15:54:54.749093,cpp,True,3375,373477635

The gh-mrva *client* sends the following.  The source is
gh-mrva/utils/utils.go,
    client.Get(fmt.Sprintf("http://localhost:8080/repos/%s/code-scanning/codeql/databases/%s", task.Nwo, task.Language))

We have
  cd /Users/hohn/work-gh/mrva/gh-mrva
  0:$ rg 'repos/.*/code-scanning/codeql/databases'

          ...
          utils/utils.go
          625:	// resp, err := client.Get(fmt.Sprintf("https://api.github.com/repos/%s/code-scanning/codeql/databases/%s", task.Nwo, task.Language))
          626:	resp, err := client.Get(fmt.Sprintf("http://localhost:8080/repos/%s/code-scanning/codeql/databases/%s", task.Nwo, task.Language))

  And
          resp, err := client.Get(fmt.Sprintf("http://localhost:8080/repos/%s/code-scanning/codeql/databases/%s", task.Nwo, task.Language))

The original DB upload was
  cd ~/work-gh/mrva/mrvacommander/client/qldbtools && \
      ./bin/mc-db-populate-minio -n 11 < scratch/db-info-3.csv

  ...
  2024-08-14 09:29:19 [INFO] Uploaded /Users/hohn/work-gh/mrva/mrva-open-source-download/repos-2024-04-29/tdlib/telegram-bot-api/code-scanning/codeql/databases/cpp/db.zip as tdlib$telegram-bot-apictsj8529d9.zip to bucket qldb
  ...
This commit is contained in:
Michael Hohn
2024-08-14 13:01:15 -07:00
committed by =Michael Hohn
parent 6bebf4abfc
commit 0a52b729cd
5 changed files with 62 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/qldbstore/interfaces.go
View File

@@ -5,11 +5,11 @@ import (
)
type CodeQLDatabaseLocation struct {
// `data` is a map of key-value pairs that describe the location of the database.
// `Data` is a map of key-value pairs that describe the location of the database.
// For example, a simple key-value pair could be "path" -> "/path/to/database.zip".
// A more complex implementation could be "bucket" -> "example", "key" -> "unique_identifier".
// XX: static types
data map[string]string
Data map[string]string
}
type Store interface {

4
pkg/qldbstore/qldbstore_local.go
View File

@@ -35,7 +35,7 @@ func (store *FilesystemCodeQLDatabaseStore) FindAvailableDBs(analysisReposReques
}
func (store *FilesystemCodeQLDatabaseStore) GetDatabase(location CodeQLDatabaseLocation) ([]byte, error) {
path, exists := location.data["path"]
path, exists := location.Data["path"]
if !exists {
return nil, fmt.Errorf("path not specified in location")
}
@@ -57,7 +57,7 @@ func (store *FilesystemCodeQLDatabaseStore) GetDatabaseLocationByNWO(nwo common.
}
location := CodeQLDatabaseLocation{
data: map[string]string{
Data: map[string]string{
"path": filePath,
},
}

14
pkg/qldbstore/qldbstore_minio.go
View File

@@ -12,7 +12,13 @@ import (
"github.com/minio/minio-go/v7/pkg/credentials"
)
const QL_DB_BUCKETNAME = "qldb"
// XX: static types: split by type?
// Restrict the keys / values and centralize the common ones here
const (
QL_DB_BUCKETNAME = "qldb"
QL_KEY_BUCKET = "bucket"
QL_KEY_KEY = "key"
)
type MinIOCodeQLDatabaseStore struct {
client *minio.Client
@@ -59,8 +65,8 @@ func (store *MinIOCodeQLDatabaseStore) FindAvailableDBs(analysisReposRequested [
}
func (store *MinIOCodeQLDatabaseStore) GetDatabase(location CodeQLDatabaseLocation) ([]byte, error) {
bucket := location.data[artifactstore.AF_KEY_BUCKET]
key := location.data[artifactstore.AF_KEY_KEY]
bucket := location.Data[artifactstore.AF_KEY_BUCKET]
key := location.Data[artifactstore.AF_KEY_KEY]
object, err := store.client.GetObject(context.Background(), bucket, key, minio.GetObjectOptions{})
if err != nil {
@@ -89,7 +95,7 @@ func (store *MinIOCodeQLDatabaseStore) GetDatabaseLocationByNWO(nwo common.NameW
}
location := CodeQLDatabaseLocation{
data: map[string]string{
Data: map[string]string{
artifactstore.AF_KEY_BUCKET: store.bucketName,
artifactstore.AF_KEY_KEY: objectName,
},

1
pkg/server/interfaces.go
View File

@@ -10,5 +10,6 @@ type CommanderAPI interface {
MRVAStatus(w http.ResponseWriter, r *http.Request)
MRVADownloadArtifactID(w http.ResponseWriter, r *http.Request)
MRVADownloadArtifact(w http.ResponseWriter, r *http.Request)
MRVADownloadQLDB(w http.ResponseWriter, r *http.Request)
MRVADownloadServe(w http.ResponseWriter, r *http.Request)
}

47
pkg/server/server.go
View File

@@ -63,7 +63,11 @@ func setupEndpoints(c CommanderAPI) {
r.HandleFunc("/repos/{owner}/{repo}/code-scanning/codeql/variant-analyses/{codeql_variant_analysis_id}", c.MRVAStatus)
r.HandleFunc("/repositories/{controller_repo_id}/code-scanning/codeql/variant-analyses/{codeql_variant_analysis_id}", c.MRVAStatusID)
// XX: Handle endpoint
// /repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp
// Endpoints for getting a URL to download artifacts
// /repos/tdlib /telegram.../code-scanning/codeql/databases/cpp
r.HandleFunc("/repos/{repo_owner}/{repo_name}/code-scanning/codeql/databases/{repo_language}", c.MRVADownloadQLDB)
r.HandleFunc("/repos/{controller_owner}/{controller_repo}/code-scanning/codeql/variant-analyses/{codeql_variant_analysis_id}/repos/{repo_owner}/{repo_name}", c.MRVADownloadArtifact)
r.HandleFunc("/repositories/{controller_repo_id}/code-scanning/codeql/variant-analyses/{codeql_variant_analysis_id}/repositories/{repository_id}", c.MRVADownloadArtifactID)
@@ -362,6 +366,49 @@ func (c *CommanderSingle) MRVADownloadArtifactID(w http.ResponseWriter, r *http.
c.MRVADownloadArtifactCommon(w, r, int(repoId), jobSpec)
}
func (c *CommanderSingle) MRVADownloadQLDB(w http.ResponseWriter, r *http.Request) {
// The repositories are uploaded without language and can be downloaded
// without it. We ignore the language parameter passed in the request:
// vars["repo_language"]
// Other artifact downloads, like sendArtifactDownloadResponse, depend on
// a jobspec (integer job id). This request has none, and needs none.
// An original upload example is
// tdlib$telegram-bot-apictsj8529d9.zip to bucket qldb.
// This is a direct data request -- don't reply with a download url.
vars := mux.Vars(r)
owner := vars["repo_owner"]
name := vars["repo_name"]
dbl := qldbstore.CodeQLDatabaseLocation{
Data: map[string]string{
qldbstore.QL_KEY_BUCKET: qldbstore.QL_DB_BUCKETNAME,
qldbstore.QL_KEY_KEY: fmt.Sprintf("%s$%s.zip", owner, name),
},
}
slog.Debug("Returning codeql database using database location",
"qldbstore.CodeQLDatabaseLocation", dbl,
)
dbContent, err := c.v.CodeQLDBStore.GetDatabase(dbl)
if err != nil {
slog.Error("Failed to retrieve ql database",
"error", err,
"qldbstore.CodeQLDatabaseLocation", dbl,
)
http.Error(w, "Failed to retrieve ql database", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/octet-stream")
w.Write(dbContent)
}
func (c *CommanderSingle) MRVADownloadArtifact(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)