Expand the codeql db download response
End-to-end testing contained an unhandled CodeQL database download
request. The handlers are added in this patch. Debugging info is below
for reference.
The mrvacommander *server* fails with the following. The source code is
: func setupEndpoints(c CommanderAPI)
See mrvacommander/pkg/server/server.go, endpoints for getting a URL to download artifacts.
Original
Downloading artifacts for tdlib_telegram-bot-apictsj8529d9_2
...
Downloading database tdlib/telegram-bot-apictsj8529d9 cpp mirva-session-1400 tdlib_telegram-bot-apictsj8529d9_2
...
2024/08/13 12:31:38 >> GET http://localhost:8080/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp
...
2024/08/13 12:31:38 << 404 http://localhost:8080/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp
...
-rwxr-xr-x@ 1 hohn staff 169488 Aug 13 12:29 tdlib_telegram-bot-apictsj8529d9_2.sarif*
-rwxr-xr-x@ 1 hohn staff 10 Aug 13 12:31 tdlib_telegram-bot-apictsj8529d9_2_db.zip*
Server log
server | 2024/08/13 19:31:38 ERROR Unhandled endpoint method=GET uri=/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp
Try a manual download from the server
8:$ wget http://localhost:8080/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp
--2024-08-13 12:56:05-- http://localhost:8080/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp
Resolving localhost (localhost)... ::1, 127.0.0.1
Connecting to localhost (localhost)|::1|:8080... connected.
HTTP request sent, awaiting response... 404 Not Found
2024-08-13 12:56:05 ERROR 404: Not Found.
server | 2024/08/13 19:56:05 ERROR Unhandled endpoint method=GET uri=/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp
The full info for the DB
tdlib,telegram-bot-api,8529d9,2.17.0,2024-05-09 08:02:49.545174+00:00,cpp,f95d406da67adb8ac13d9c562291aa57c65398e0,306106.0,/Users/hohn/work-gh/mrva/mrva-open-source-download/repos-2024-04-29/tdlib/telegram-bot-api/code-scanning/codeql/databases/cpp/db.zip,cpp,C/C++,1244.0,306106.0,2024-05-13T15:54:54.749093,cpp,True,3375,373477635
The gh-mrva *client* sends the following. The source is
gh-mrva/utils/utils.go,
client.Get(fmt.Sprintf("http://localhost:8080/repos/%s/code-scanning/codeql/databases/%s", task.Nwo, task.Language))
We have
cd /Users/hohn/work-gh/mrva/gh-mrva
0:$ rg 'repos/.*/code-scanning/codeql/databases'
...
utils/utils.go
625: // resp, err := client.Get(fmt.Sprintf("https://api.github.com/repos/%s/code-scanning/codeql/databases/%s", task.Nwo, task.Language))
626: resp, err := client.Get(fmt.Sprintf("http://localhost:8080/repos/%s/code-scanning/codeql/databases/%s", task.Nwo, task.Language))
And
resp, err := client.Get(fmt.Sprintf("http://localhost:8080/repos/%s/code-scanning/codeql/databases/%s", task.Nwo, task.Language))
The original DB upload was
cd ~/work-gh/mrva/mrvacommander/client/qldbtools && \
./bin/mc-db-populate-minio -n 11 < scratch/db-info-3.csv
...
2024-08-14 09:29:19 [INFO] Uploaded /Users/hohn/work-gh/mrva/mrva-open-source-download/repos-2024-04-29/tdlib/telegram-bot-api/code-scanning/codeql/databases/cpp/db.zip as tdlib$telegram-bot-apictsj8529d9.zip to bucket qldb
...
This commit is contained in:
committed by
=Michael Hohn
parent
6bebf4abfc
commit
0a52b729cd
@@ -5,11 +5,11 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type CodeQLDatabaseLocation struct {
|
type CodeQLDatabaseLocation struct {
|
||||||
// `data` is a map of key-value pairs that describe the location of the database.
|
// `Data` is a map of key-value pairs that describe the location of the database.
|
||||||
// For example, a simple key-value pair could be "path" -> "/path/to/database.zip".
|
// For example, a simple key-value pair could be "path" -> "/path/to/database.zip".
|
||||||
// A more complex implementation could be "bucket" -> "example", "key" -> "unique_identifier".
|
// A more complex implementation could be "bucket" -> "example", "key" -> "unique_identifier".
|
||||||
// XX: static types
|
// XX: static types
|
||||||
data map[string]string
|
Data map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
type Store interface {
|
type Store interface {
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ func (store *FilesystemCodeQLDatabaseStore) FindAvailableDBs(analysisReposReques
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (store *FilesystemCodeQLDatabaseStore) GetDatabase(location CodeQLDatabaseLocation) ([]byte, error) {
|
func (store *FilesystemCodeQLDatabaseStore) GetDatabase(location CodeQLDatabaseLocation) ([]byte, error) {
|
||||||
path, exists := location.data["path"]
|
path, exists := location.Data["path"]
|
||||||
if !exists {
|
if !exists {
|
||||||
return nil, fmt.Errorf("path not specified in location")
|
return nil, fmt.Errorf("path not specified in location")
|
||||||
}
|
}
|
||||||
@@ -57,7 +57,7 @@ func (store *FilesystemCodeQLDatabaseStore) GetDatabaseLocationByNWO(nwo common.
|
|||||||
}
|
}
|
||||||
|
|
||||||
location := CodeQLDatabaseLocation{
|
location := CodeQLDatabaseLocation{
|
||||||
data: map[string]string{
|
Data: map[string]string{
|
||||||
"path": filePath,
|
"path": filePath,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,7 +12,13 @@ import (
|
|||||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||||
)
|
)
|
||||||
|
|
||||||
const QL_DB_BUCKETNAME = "qldb"
|
// XX: static types: split by type?
|
||||||
|
// Restrict the keys / values and centralize the common ones here
|
||||||
|
const (
|
||||||
|
QL_DB_BUCKETNAME = "qldb"
|
||||||
|
QL_KEY_BUCKET = "bucket"
|
||||||
|
QL_KEY_KEY = "key"
|
||||||
|
)
|
||||||
|
|
||||||
type MinIOCodeQLDatabaseStore struct {
|
type MinIOCodeQLDatabaseStore struct {
|
||||||
client *minio.Client
|
client *minio.Client
|
||||||
@@ -59,8 +65,8 @@ func (store *MinIOCodeQLDatabaseStore) FindAvailableDBs(analysisReposRequested [
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (store *MinIOCodeQLDatabaseStore) GetDatabase(location CodeQLDatabaseLocation) ([]byte, error) {
|
func (store *MinIOCodeQLDatabaseStore) GetDatabase(location CodeQLDatabaseLocation) ([]byte, error) {
|
||||||
bucket := location.data[artifactstore.AF_KEY_BUCKET]
|
bucket := location.Data[artifactstore.AF_KEY_BUCKET]
|
||||||
key := location.data[artifactstore.AF_KEY_KEY]
|
key := location.Data[artifactstore.AF_KEY_KEY]
|
||||||
|
|
||||||
object, err := store.client.GetObject(context.Background(), bucket, key, minio.GetObjectOptions{})
|
object, err := store.client.GetObject(context.Background(), bucket, key, minio.GetObjectOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -89,7 +95,7 @@ func (store *MinIOCodeQLDatabaseStore) GetDatabaseLocationByNWO(nwo common.NameW
|
|||||||
}
|
}
|
||||||
|
|
||||||
location := CodeQLDatabaseLocation{
|
location := CodeQLDatabaseLocation{
|
||||||
data: map[string]string{
|
Data: map[string]string{
|
||||||
artifactstore.AF_KEY_BUCKET: store.bucketName,
|
artifactstore.AF_KEY_BUCKET: store.bucketName,
|
||||||
artifactstore.AF_KEY_KEY: objectName,
|
artifactstore.AF_KEY_KEY: objectName,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -10,5 +10,6 @@ type CommanderAPI interface {
|
|||||||
MRVAStatus(w http.ResponseWriter, r *http.Request)
|
MRVAStatus(w http.ResponseWriter, r *http.Request)
|
||||||
MRVADownloadArtifactID(w http.ResponseWriter, r *http.Request)
|
MRVADownloadArtifactID(w http.ResponseWriter, r *http.Request)
|
||||||
MRVADownloadArtifact(w http.ResponseWriter, r *http.Request)
|
MRVADownloadArtifact(w http.ResponseWriter, r *http.Request)
|
||||||
|
MRVADownloadQLDB(w http.ResponseWriter, r *http.Request)
|
||||||
MRVADownloadServe(w http.ResponseWriter, r *http.Request)
|
MRVADownloadServe(w http.ResponseWriter, r *http.Request)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -63,7 +63,11 @@ func setupEndpoints(c CommanderAPI) {
|
|||||||
r.HandleFunc("/repos/{owner}/{repo}/code-scanning/codeql/variant-analyses/{codeql_variant_analysis_id}", c.MRVAStatus)
|
r.HandleFunc("/repos/{owner}/{repo}/code-scanning/codeql/variant-analyses/{codeql_variant_analysis_id}", c.MRVAStatus)
|
||||||
r.HandleFunc("/repositories/{controller_repo_id}/code-scanning/codeql/variant-analyses/{codeql_variant_analysis_id}", c.MRVAStatusID)
|
r.HandleFunc("/repositories/{controller_repo_id}/code-scanning/codeql/variant-analyses/{codeql_variant_analysis_id}", c.MRVAStatusID)
|
||||||
|
|
||||||
|
// XX: Handle endpoint
|
||||||
|
// /repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp
|
||||||
// Endpoints for getting a URL to download artifacts
|
// Endpoints for getting a URL to download artifacts
|
||||||
|
// /repos/tdlib /telegram.../code-scanning/codeql/databases/cpp
|
||||||
|
r.HandleFunc("/repos/{repo_owner}/{repo_name}/code-scanning/codeql/databases/{repo_language}", c.MRVADownloadQLDB)
|
||||||
r.HandleFunc("/repos/{controller_owner}/{controller_repo}/code-scanning/codeql/variant-analyses/{codeql_variant_analysis_id}/repos/{repo_owner}/{repo_name}", c.MRVADownloadArtifact)
|
r.HandleFunc("/repos/{controller_owner}/{controller_repo}/code-scanning/codeql/variant-analyses/{codeql_variant_analysis_id}/repos/{repo_owner}/{repo_name}", c.MRVADownloadArtifact)
|
||||||
r.HandleFunc("/repositories/{controller_repo_id}/code-scanning/codeql/variant-analyses/{codeql_variant_analysis_id}/repositories/{repository_id}", c.MRVADownloadArtifactID)
|
r.HandleFunc("/repositories/{controller_repo_id}/code-scanning/codeql/variant-analyses/{codeql_variant_analysis_id}/repositories/{repository_id}", c.MRVADownloadArtifactID)
|
||||||
|
|
||||||
@@ -362,6 +366,49 @@ func (c *CommanderSingle) MRVADownloadArtifactID(w http.ResponseWriter, r *http.
|
|||||||
c.MRVADownloadArtifactCommon(w, r, int(repoId), jobSpec)
|
c.MRVADownloadArtifactCommon(w, r, int(repoId), jobSpec)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *CommanderSingle) MRVADownloadQLDB(w http.ResponseWriter, r *http.Request) {
|
||||||
|
// The repositories are uploaded without language and can be downloaded
|
||||||
|
// without it. We ignore the language parameter passed in the request:
|
||||||
|
// vars["repo_language"]
|
||||||
|
|
||||||
|
// Other artifact downloads, like sendArtifactDownloadResponse, depend on
|
||||||
|
// a jobspec (integer job id). This request has none, and needs none.
|
||||||
|
|
||||||
|
// An original upload example is
|
||||||
|
// tdlib$telegram-bot-apictsj8529d9.zip to bucket qldb.
|
||||||
|
|
||||||
|
// This is a direct data request -- don't reply with a download url.
|
||||||
|
|
||||||
|
vars := mux.Vars(r)
|
||||||
|
owner := vars["repo_owner"]
|
||||||
|
name := vars["repo_name"]
|
||||||
|
|
||||||
|
dbl := qldbstore.CodeQLDatabaseLocation{
|
||||||
|
Data: map[string]string{
|
||||||
|
qldbstore.QL_KEY_BUCKET: qldbstore.QL_DB_BUCKETNAME,
|
||||||
|
qldbstore.QL_KEY_KEY: fmt.Sprintf("%s$%s.zip", owner, name),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
slog.Debug("Returning codeql database using database location",
|
||||||
|
"qldbstore.CodeQLDatabaseLocation", dbl,
|
||||||
|
)
|
||||||
|
|
||||||
|
dbContent, err := c.v.CodeQLDBStore.GetDatabase(dbl)
|
||||||
|
if err != nil {
|
||||||
|
slog.Error("Failed to retrieve ql database",
|
||||||
|
"error", err,
|
||||||
|
"qldbstore.CodeQLDatabaseLocation", dbl,
|
||||||
|
)
|
||||||
|
http.Error(w, "Failed to retrieve ql database", http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", "application/octet-stream")
|
||||||
|
w.Write(dbContent)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
func (c *CommanderSingle) MRVADownloadArtifact(w http.ResponseWriter, r *http.Request) {
|
func (c *CommanderSingle) MRVADownloadArtifact(w http.ResponseWriter, r *http.Request) {
|
||||||
vars := mux.Vars(r)
|
vars := mux.Vars(r)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user