mrva/gh-mrva
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Store Query Id for each run

Browse Source
This commit is contained in:
Alvaro Muñoz
2023-09-11 16:51:45 +02:00
parent 1809c9bced
commit 81de79e4c8
4 changed files with 73 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
cmd/status.go
View File

@@ -87,6 +87,7 @@ func sessionStatus() {
results.Runs = append(results.Runs, models.RunStatus{
Id: run.Id,
Query: run.Query,
QueryId: run.QueryId,
Status: status,
FailureReason: failure_reason,
})
@@ -99,10 +100,12 @@ func sessionStatus() {
results.TotalFindingsCount += int(repo.(map[string]interface{})["result_count"].(float64))
repoInfo := repo.(map[string]interface{})["repository"].(map[string]interface{})
results.ResositoriesWithFindings = append(results.ResositoriesWithFindings, models.RepoWithFindings{
Nwo: repoInfo["full_name"].(string),
Count: int(repo.(map[string]interface{})["result_count"].(float64)),
RunId: run.Id,
Stars: int(repoInfo["stargazers_count"].(float64)),
Nwo: repoInfo["full_name"].(string),
Query: run.Query,
QueryId: run.QueryId,
Count: int(repo.(map[string]interface{})["result_count"].(float64)),
RunId: run.Id,
Stars: int(repoInfo["stargazers_count"].(float64)),
})
}
} else if repo.(map[string]interface{})["analysis_status"].(string) == "failed" {
@@ -149,7 +152,7 @@ func sessionStatus() {
fmt.Println("Total findings:", results.TotalFindingsCount)
fmt.Println("Repositories with findings:")
for _, repo := range results.ResositoriesWithFindings {
fmt.Println(" ", repo.Nwo, ":", repo.Count)
fmt.Printf(" %s (%s): %d\n", repo.Nwo, repo.QueryId, repo.Count)
}
}
}

6
cmd/submit.go
View File

@@ -126,11 +126,11 @@ func submitQuery() {
fmt.Printf("Submitting %d queries for %d repositories\n", len(queries), len(repositories))
var runs []models.Run
for _, query := range queries {
encodedBundle, err := utils.GenerateQueryPack(codeqlPath, query, language)
encodedBundle, queryId, err := utils.GenerateQueryPack(codeqlPath, query, language)
if err != nil {
log.Fatal(err)
}
fmt.Printf("Generated encoded bundle for %s\n", query)
fmt.Printf("Generated encoded bundle for %s (%s)\n", query, queryId)
var chunks [][]string
for i := 0; i < len(repositories); i += config.MAX_MRVA_REPOSITORIES {
@@ -145,7 +145,7 @@ func submitQuery() {
if err != nil {
log.Fatal(err)
}
runs = append(runs, models.Run{Id: id, Query: query})
runs = append(runs, models.Run{Id: id, Query: query, QueryId: queryId})
}
}

24
models/models.go
View File

@@ -5,8 +5,9 @@ import (
)
type Run struct {
Id int `yaml:"id"`
Query string `yaml:"query"`
Id int `yaml:"id"`
Query string `yaml:"query"`
QueryId string `yaml:"query_id"`
}
type Session struct {
@@ -36,17 +37,20 @@ type DownloadTask struct {
}
type RunStatus struct {
Id int `json:"id"`
Query string `json:"query"`
Status string `json:"status"`
FailureReason string `json:"failure_reason"`
Id int `json:"id"`
Query string `json:"query"`
QueryId string `json:"query_id"`
Status string `json:"status"`
FailureReason string `json:"failure_reason"`
}
type RepoWithFindings struct {
Nwo string `json:"nwo"`
Count int `json:"count"`
RunId int `json:"run_id"`
Stars int `json:"stars"`
Query string `json:"query"`
QueryId string `json:"query_id"`
Nwo string `json:"nwo"`
Count int `json:"count"`
RunId int `json:"run_id"`
Stars int `json:"stars"`
}
type Results struct {

54
utils/utils.go
View File

@@ -217,6 +217,44 @@ func ResolveRepositories(listFile string, list string) ([]string, error) {
return repoLists[list], nil
}
func ResolveQueryId(queryFile string) (string, error) {
// read contents of queryFile into a string
queryFile, err := filepath.Abs(queryFile)
if err != nil {
log.Fatal(err)
}
if _, err := os.Stat(queryFile); errors.Is(err, os.ErrNotExist) {
log.Fatal(fmt.Sprintf("Query file %s does not exist", queryFile))
}
file, err := os.Open(queryFile)
if err != nil {
log.Fatal(err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
scanner.Split(bufio.ScanLines)
var fileLines []string
for scanner.Scan() {
fileLines = append(fileLines, scanner.Text())
}
if err := scanner.Err(); err != nil {
log.Fatal(err)
}
// find the query id which is a line that starts with ` * @id QUERY_ID`
queryId := ""
for _, line := range fileLines {
if strings.HasPrefix(line, " * @id ") {
queryId = strings.TrimPrefix(line, " * @id ")
break
}
}
fmt.Println("Query ID:", queryId)
if queryId == "" {
return "", errors.New("Failed to find query id in query file")
}
return queryId, nil
}
func ResolveQueries(codeqlPath string, querySuite string) []string {
args := []string{"resolve", "queries", "--format=json", querySuite}
jsonBytes, err := RunCodeQLCommand(codeqlPath, false, args...)
@@ -246,7 +284,7 @@ func RunCodeQLCommand(codeqlPath string, combined bool, args ...string) ([]byte,
}
}
func GenerateQueryPack(codeqlPath string, queryFile string, language string) (string, error) {
func GenerateQueryPack(codeqlPath string, queryFile string, language string) (string, string, error) {
fmt.Printf("Generating query pack for %s\n", queryFile)
// create a temporary directory to hold the query pack
@@ -263,6 +301,10 @@ func GenerateQueryPack(codeqlPath string, queryFile string, language string) (st
if _, err := os.Stat(queryFile); errors.Is(err, os.ErrNotExist) {
log.Fatal(fmt.Sprintf("Query file %s does not exist", queryFile))
}
queryId, err := ResolveQueryId(queryFile)
if err != nil {
log.Fatal(err)
}
originalPackRoot := FindPackRoot(queryFile)
packRelativePath, _ := filepath.Rel(originalPackRoot, queryFile)
targetQueryFileName := filepath.Join(queryPackDir, packRelativePath)
@@ -346,7 +388,7 @@ defaultSuite:
stdouterr, err := RunCodeQLCommand(codeqlPath, true, args...)
if err != nil {
fmt.Printf("`codeql pack bundle` failed with error: %v\n", string(stdouterr))
return "", fmt.Errorf("Failed to install query pack: %v", err)
return "", "", fmt.Errorf("Failed to install query pack: %v", err)
}
// bundle the query pack
fmt.Print("Compiling and bundling the QLPack (This may take a while)\n")
@@ -355,22 +397,22 @@ defaultSuite:
stdouterr, err = RunCodeQLCommand(codeqlPath, true, args...)
if err != nil {
fmt.Printf("`codeql pack bundle` failed with error: %v\n", string(stdouterr))
return "", fmt.Errorf("Failed to bundle query pack: %v\n", err)
return "", "", fmt.Errorf("Failed to bundle query pack: %v\n", err)
}
// open the bundle file and encode it as base64
bundleFile, err := os.Open(bundlePath)
if err != nil {
return "", fmt.Errorf("Failed to open bundle file: %v\n", err)
return "", "", fmt.Errorf("Failed to open bundle file: %v\n", err)
}
defer bundleFile.Close()
bundleBytes, err := ioutil.ReadAll(bundleFile)
if err != nil {
return "", fmt.Errorf("Failed to read bundle file: %v\n", err)
return "", "", fmt.Errorf("Failed to read bundle file: %v\n", err)
}
bundleBase64 := base64.StdEncoding.EncodeToString(bundleBytes)
return bundleBase64, nil
return bundleBase64, queryId, nil
}