End-to-end testing contained an unhandled CodeQL database download
request. The handlers are added in this patch. Debugging info is below
for reference.
The mrvacommander *server* fails with the following. The source code is
: func setupEndpoints(c CommanderAPI)
See mrvacommander/pkg/server/server.go, endpoints for getting a URL to download artifacts.
Original
Downloading artifacts for tdlib_telegram-bot-apictsj8529d9_2
...
Downloading database tdlib/telegram-bot-apictsj8529d9 cpp mirva-session-1400 tdlib_telegram-bot-apictsj8529d9_2
...
2024/08/13 12:31:38 >> GET http://localhost:8080/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp
...
2024/08/13 12:31:38 << 404 http://localhost:8080/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp
...
-rwxr-xr-x@ 1 hohn staff 169488 Aug 13 12:29 tdlib_telegram-bot-apictsj8529d9_2.sarif*
-rwxr-xr-x@ 1 hohn staff 10 Aug 13 12:31 tdlib_telegram-bot-apictsj8529d9_2_db.zip*
Server log
server | 2024/08/13 19:31:38 ERROR Unhandled endpoint method=GET uri=/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp
Try a manual download from the server
8:$ wget http://localhost:8080/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp
--2024-08-13 12:56:05-- http://localhost:8080/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp
Resolving localhost (localhost)... ::1, 127.0.0.1
Connecting to localhost (localhost)|::1|:8080... connected.
HTTP request sent, awaiting response... 404 Not Found
2024-08-13 12:56:05 ERROR 404: Not Found.
server | 2024/08/13 19:56:05 ERROR Unhandled endpoint method=GET uri=/repos/tdlib/telegram-bot-apictsj8529d9/code-scanning/codeql/databases/cpp
The full info for the DB
tdlib,telegram-bot-api,8529d9,2.17.0,2024-05-09 08:02:49.545174+00:00,cpp,f95d406da67adb8ac13d9c562291aa57c65398e0,306106.0,/Users/hohn/work-gh/mrva/mrva-open-source-download/repos-2024-04-29/tdlib/telegram-bot-api/code-scanning/codeql/databases/cpp/db.zip,cpp,C/C++,1244.0,306106.0,2024-05-13T15:54:54.749093,cpp,True,3375,373477635
The gh-mrva *client* sends the following. The source is
gh-mrva/utils/utils.go,
client.Get(fmt.Sprintf("http://localhost:8080/repos/%s/code-scanning/codeql/databases/%s", task.Nwo, task.Language))
We have
cd /Users/hohn/work-gh/mrva/gh-mrva
0:$ rg 'repos/.*/code-scanning/codeql/databases'
...
utils/utils.go
625: // resp, err := client.Get(fmt.Sprintf("https://api.github.com/repos/%s/code-scanning/codeql/databases/%s", task.Nwo, task.Language))
626: resp, err := client.Get(fmt.Sprintf("http://localhost:8080/repos/%s/code-scanning/codeql/databases/%s", task.Nwo, task.Language))
And
resp, err := client.Get(fmt.Sprintf("http://localhost:8080/repos/%s/code-scanning/codeql/databases/%s", task.Nwo, task.Language))
The original DB upload was
cd ~/work-gh/mrva/mrvacommander/client/qldbtools && \
./bin/mc-db-populate-minio -n 11 < scratch/db-info-3.csv
...
2024-08-14 09:29:19 [INFO] Uploaded /Users/hohn/work-gh/mrva/mrva-open-source-download/repos-2024-04-29/tdlib/telegram-bot-api/code-scanning/codeql/databases/cpp/db.zip as tdlib$telegram-bot-apictsj8529d9.zip to bucket qldb
...
67 lines
1.7 KiB
Go
67 lines
1.7 KiB
Go
package qldbstore
|
|
|
|
import (
|
|
"fmt"
|
|
"mrvacommander/pkg/common"
|
|
"os"
|
|
"path/filepath"
|
|
)
|
|
|
|
type FilesystemCodeQLDatabaseStore struct {
|
|
basePath string
|
|
}
|
|
|
|
func NewLocalFilesystemCodeQLDatabaseStore(basePath string) *FilesystemCodeQLDatabaseStore {
|
|
return &FilesystemCodeQLDatabaseStore{
|
|
basePath: basePath,
|
|
}
|
|
}
|
|
|
|
func (store *FilesystemCodeQLDatabaseStore) FindAvailableDBs(analysisReposRequested []common.NameWithOwner) (
|
|
notFoundRepos []common.NameWithOwner,
|
|
foundRepos *map[common.NameWithOwner]CodeQLDatabaseLocation) {
|
|
|
|
foundReposMap := make(map[common.NameWithOwner]CodeQLDatabaseLocation)
|
|
for _, repo := range analysisReposRequested {
|
|
location, err := store.GetDatabaseLocationByNWO(repo)
|
|
if err != nil {
|
|
notFoundRepos = append(notFoundRepos, repo)
|
|
} else {
|
|
foundReposMap[repo] = location
|
|
}
|
|
}
|
|
|
|
return notFoundRepos, &foundReposMap
|
|
}
|
|
|
|
func (store *FilesystemCodeQLDatabaseStore) GetDatabase(location CodeQLDatabaseLocation) ([]byte, error) {
|
|
path, exists := location.Data["path"]
|
|
if !exists {
|
|
return nil, fmt.Errorf("path not specified in location")
|
|
}
|
|
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return data, nil
|
|
}
|
|
|
|
func (store *FilesystemCodeQLDatabaseStore) GetDatabaseLocationByNWO(nwo common.NameWithOwner) (CodeQLDatabaseLocation, error) {
|
|
filePath := filepath.Join(store.basePath, fmt.Sprintf("%s/%s/%s_%s_db.zip", nwo.Owner, nwo.Repo, nwo.Owner, nwo.Repo))
|
|
|
|
// Check if the file exists
|
|
if _, err := os.Stat(filePath); os.IsNotExist(err) {
|
|
return CodeQLDatabaseLocation{}, fmt.Errorf("database not found for %s", nwo)
|
|
}
|
|
|
|
location := CodeQLDatabaseLocation{
|
|
Data: map[string]string{
|
|
"path": filePath,
|
|
},
|
|
}
|
|
|
|
return location, nil
|
|
}
|