From 81de79e4c80535830a67f8e50e20ab9b4f06d162 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alvaro=20Mu=C3=B1oz?= Date: Mon, 11 Sep 2023 16:51:45 +0200 Subject: [PATCH] Store Query Id for each run --- cmd/status.go | 13 +++++++----- cmd/submit.go | 6 +++--- models/models.go | 24 ++++++++++++--------- utils/utils.go | 54 ++++++++++++++++++++++++++++++++++++++++++------ 4 files changed, 73 insertions(+), 24 deletions(-) diff --git a/cmd/status.go b/cmd/status.go index 81c6b68..1db5de7 100644 --- a/cmd/status.go +++ b/cmd/status.go @@ -87,6 +87,7 @@ func sessionStatus() { results.Runs = append(results.Runs, models.RunStatus{ Id: run.Id, Query: run.Query, + QueryId: run.QueryId, Status: status, FailureReason: failure_reason, }) @@ -99,10 +100,12 @@ func sessionStatus() { results.TotalFindingsCount += int(repo.(map[string]interface{})["result_count"].(float64)) repoInfo := repo.(map[string]interface{})["repository"].(map[string]interface{}) results.ResositoriesWithFindings = append(results.ResositoriesWithFindings, models.RepoWithFindings{ - Nwo: repoInfo["full_name"].(string), - Count: int(repo.(map[string]interface{})["result_count"].(float64)), - RunId: run.Id, - Stars: int(repoInfo["stargazers_count"].(float64)), + Nwo: repoInfo["full_name"].(string), + Query: run.Query, + QueryId: run.QueryId, + Count: int(repo.(map[string]interface{})["result_count"].(float64)), + RunId: run.Id, + Stars: int(repoInfo["stargazers_count"].(float64)), }) } } else if repo.(map[string]interface{})["analysis_status"].(string) == "failed" { @@ -149,7 +152,7 @@ func sessionStatus() { fmt.Println("Total findings:", results.TotalFindingsCount) fmt.Println("Repositories with findings:") for _, repo := range results.ResositoriesWithFindings { - fmt.Println(" ", repo.Nwo, ":", repo.Count) + fmt.Printf(" %s (%s): %d\n", repo.Nwo, repo.QueryId, repo.Count) } } } diff --git a/cmd/submit.go b/cmd/submit.go index 23781fa..571b7b3 100644 --- a/cmd/submit.go +++ b/cmd/submit.go @@ -126,11 +126,11 @@ func submitQuery() { fmt.Printf("Submitting %d queries for %d repositories\n", len(queries), len(repositories)) var runs []models.Run for _, query := range queries { - encodedBundle, err := utils.GenerateQueryPack(codeqlPath, query, language) + encodedBundle, queryId, err := utils.GenerateQueryPack(codeqlPath, query, language) if err != nil { log.Fatal(err) } - fmt.Printf("Generated encoded bundle for %s\n", query) + fmt.Printf("Generated encoded bundle for %s (%s)\n", query, queryId) var chunks [][]string for i := 0; i < len(repositories); i += config.MAX_MRVA_REPOSITORIES { @@ -145,7 +145,7 @@ func submitQuery() { if err != nil { log.Fatal(err) } - runs = append(runs, models.Run{Id: id, Query: query}) + runs = append(runs, models.Run{Id: id, Query: query, QueryId: queryId}) } } diff --git a/models/models.go b/models/models.go index 5898c82..db36125 100644 --- a/models/models.go +++ b/models/models.go @@ -5,8 +5,9 @@ import ( ) type Run struct { - Id int `yaml:"id"` - Query string `yaml:"query"` + Id int `yaml:"id"` + Query string `yaml:"query"` + QueryId string `yaml:"query_id"` } type Session struct { @@ -36,17 +37,20 @@ type DownloadTask struct { } type RunStatus struct { - Id int `json:"id"` - Query string `json:"query"` - Status string `json:"status"` - FailureReason string `json:"failure_reason"` + Id int `json:"id"` + Query string `json:"query"` + QueryId string `json:"query_id"` + Status string `json:"status"` + FailureReason string `json:"failure_reason"` } type RepoWithFindings struct { - Nwo string `json:"nwo"` - Count int `json:"count"` - RunId int `json:"run_id"` - Stars int `json:"stars"` + Query string `json:"query"` + QueryId string `json:"query_id"` + Nwo string `json:"nwo"` + Count int `json:"count"` + RunId int `json:"run_id"` + Stars int `json:"stars"` } type Results struct { diff --git a/utils/utils.go b/utils/utils.go index f5dea95..c633550 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -217,6 +217,44 @@ func ResolveRepositories(listFile string, list string) ([]string, error) { return repoLists[list], nil } +func ResolveQueryId(queryFile string) (string, error) { + // read contents of queryFile into a string + queryFile, err := filepath.Abs(queryFile) + if err != nil { + log.Fatal(err) + } + if _, err := os.Stat(queryFile); errors.Is(err, os.ErrNotExist) { + log.Fatal(fmt.Sprintf("Query file %s does not exist", queryFile)) + } + file, err := os.Open(queryFile) + if err != nil { + log.Fatal(err) + } + defer file.Close() + scanner := bufio.NewScanner(file) + scanner.Split(bufio.ScanLines) + var fileLines []string + for scanner.Scan() { + fileLines = append(fileLines, scanner.Text()) + } + if err := scanner.Err(); err != nil { + log.Fatal(err) + } + // find the query id which is a line that starts with ` * @id QUERY_ID` + queryId := "" + for _, line := range fileLines { + if strings.HasPrefix(line, " * @id ") { + queryId = strings.TrimPrefix(line, " * @id ") + break + } + } + fmt.Println("Query ID:", queryId) + if queryId == "" { + return "", errors.New("Failed to find query id in query file") + } + return queryId, nil +} + func ResolveQueries(codeqlPath string, querySuite string) []string { args := []string{"resolve", "queries", "--format=json", querySuite} jsonBytes, err := RunCodeQLCommand(codeqlPath, false, args...) @@ -246,7 +284,7 @@ func RunCodeQLCommand(codeqlPath string, combined bool, args ...string) ([]byte, } } -func GenerateQueryPack(codeqlPath string, queryFile string, language string) (string, error) { +func GenerateQueryPack(codeqlPath string, queryFile string, language string) (string, string, error) { fmt.Printf("Generating query pack for %s\n", queryFile) // create a temporary directory to hold the query pack @@ -263,6 +301,10 @@ func GenerateQueryPack(codeqlPath string, queryFile string, language string) (st if _, err := os.Stat(queryFile); errors.Is(err, os.ErrNotExist) { log.Fatal(fmt.Sprintf("Query file %s does not exist", queryFile)) } + queryId, err := ResolveQueryId(queryFile) + if err != nil { + log.Fatal(err) + } originalPackRoot := FindPackRoot(queryFile) packRelativePath, _ := filepath.Rel(originalPackRoot, queryFile) targetQueryFileName := filepath.Join(queryPackDir, packRelativePath) @@ -346,7 +388,7 @@ defaultSuite: stdouterr, err := RunCodeQLCommand(codeqlPath, true, args...) if err != nil { fmt.Printf("`codeql pack bundle` failed with error: %v\n", string(stdouterr)) - return "", fmt.Errorf("Failed to install query pack: %v", err) + return "", "", fmt.Errorf("Failed to install query pack: %v", err) } // bundle the query pack fmt.Print("Compiling and bundling the QLPack (This may take a while)\n") @@ -355,22 +397,22 @@ defaultSuite: stdouterr, err = RunCodeQLCommand(codeqlPath, true, args...) if err != nil { fmt.Printf("`codeql pack bundle` failed with error: %v\n", string(stdouterr)) - return "", fmt.Errorf("Failed to bundle query pack: %v\n", err) + return "", "", fmt.Errorf("Failed to bundle query pack: %v\n", err) } // open the bundle file and encode it as base64 bundleFile, err := os.Open(bundlePath) if err != nil { - return "", fmt.Errorf("Failed to open bundle file: %v\n", err) + return "", "", fmt.Errorf("Failed to open bundle file: %v\n", err) } defer bundleFile.Close() bundleBytes, err := ioutil.ReadAll(bundleFile) if err != nil { - return "", fmt.Errorf("Failed to read bundle file: %v\n", err) + return "", "", fmt.Errorf("Failed to read bundle file: %v\n", err) } bundleBase64 := base64.StdEncoding.EncodeToString(bundleBytes) - return bundleBase64, nil + return bundleBase64, queryId, nil }