Introduce explicit type QueryLanguage = string and update code to clarify

Previously:
- There is confusion between nameWithOwner and queryLanguage.  Both are strings.
  Between

        runResult, err := codeql.RunQuery(databasePath, job.QueryLanguage, queryPackPath, tempDir)
    (agent.go l205)

  and

        func RunQuery(database string, nwo string, queryPackPath string, tempDir string) (*RunQueryResult, error)

  QueryLanguage is suddenly name with owner in the code.

  Added some debugging, the value is the query language in the two places it gets used:

        server         | 2024/07/03 18:30:15 DEBUG Processed request info location="{Data:map[bucket:packs key:1]}" language=cpp
        ...
        agent          | 2024/07/03 18:30:15 DEBUG XX: is nwo a name/owner, or the original callers' queryLanguage? nwo=cpp
        ...
        agent          | 2024/07/03 18:30:19 DEBUG XX: 2: is nwo a name/owner, or the original callers' queryLanguage? nwo=cpp

Changes:
- Introduce explicit type QueryLanguage = string and update code to clarify
- inline trivial function
This commit is contained in:
Michael Hohn
2024-07-03 13:30:02 -07:00
committed by =Michael Hohn
parent 07f93f3d27
commit b3cf7a4f65
4 changed files with 38 additions and 19 deletions

View File

@@ -8,6 +8,7 @@ import (
"io" "io"
"log" "log"
"log/slog" "log/slog"
"mrvacommander/pkg/queue"
"mrvacommander/utils" "mrvacommander/utils"
"os" "os"
"os/exec" "os/exec"
@@ -81,8 +82,11 @@ func addFileToZip(zipWriter *zip.Writer, filePath, zipPath string) error {
return nil return nil
} }
func RunQuery(database string, nwo string, queryPackPath string, tempDir string) (*RunQueryResult, error) { func RunQuery(database string, language queue.QueryLanguage, queryPackPath string, tempDir string) (*RunQueryResult, error) {
path, err := getCodeQLCLIPath() path, err := getCodeQLCLIPath()
// XX: is nwo a name/owner, or the original callers' queryLanguage?
slog.Debug("XX: is nwo a name/owner, or the original callers' queryLanguage?",
"language", language)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to get codeql cli path: %v", err) return nil, fmt.Errorf("failed to get codeql cli path: %v", err)
@@ -142,7 +146,7 @@ func RunQuery(database string, nwo string, queryPackPath string, tempDir string)
var sarifFilePath string var sarifFilePath string
if shouldGenerateSarif { if shouldGenerateSarif {
sarif, err := generateSarif(codeql, nwo, databasePath, queryPackPath, databaseSHA, resultsDir) sarif, err := generateSarif(codeql, language, databasePath, queryPackPath, databaseSHA, resultsDir)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to generate SARIF: %v", err) return nil, fmt.Errorf("failed to generate SARIF: %v", err)
} }
@@ -347,7 +351,7 @@ func getSarifOutputType(queryMetadata QueryMetadata, compatibleQueryKinds []stri
return "" return ""
} }
func generateSarif(codeql CodeqlCli, nwo, databasePath, queryPackPath, databaseSHA string, resultsDir string) ([]byte, error) { func generateSarif(codeql CodeqlCli, language queue.QueryLanguage, databasePath, queryPackPath, databaseSHA string, resultsDir string) ([]byte, error) {
sarifFile := filepath.Join(resultsDir, "results.sarif") sarifFile := filepath.Join(resultsDir, "results.sarif")
cmd := exec.Command(codeql.Path, "database", "interpret-results", "--format=sarif-latest", "--output="+sarifFile, "--sarif-add-snippets", "--no-group-results", databasePath, queryPackPath) cmd := exec.Command(codeql.Path, "database", "interpret-results", "--format=sarif-latest", "--output="+sarifFile, "--sarif-add-snippets", "--no-group-results", databasePath, queryPackPath)
if output, err := cmd.CombinedOutput(); err != nil { if output, err := cmd.CombinedOutput(); err != nil {
@@ -359,12 +363,21 @@ func generateSarif(codeql CodeqlCli, nwo, databasePath, queryPackPath, databaseS
return nil, fmt.Errorf("failed to read SARIF file: %v", err) return nil, fmt.Errorf("failed to read SARIF file: %v", err)
} }
// Modify the sarif: start by extracting
var sarif Sarif var sarif Sarif
if err := json.Unmarshal(sarifData, &sarif); err != nil { if err := json.Unmarshal(sarifData, &sarif); err != nil {
return nil, fmt.Errorf("failed to unmarshal SARIF: %v", err) return nil, fmt.Errorf("failed to unmarshal SARIF: %v", err)
} }
injectVersionControlInfo(&sarif, nwo, databaseSHA) // now inject version control info
for _, run := range sarif.Runs {
run.VersionControlProvenance = append(run.VersionControlProvenance, map[string]interface{}{
"repositoryUri": fmt.Sprintf("%s/%s", os.Getenv("GITHUB_SERVER_URL"), language),
"revisionId": databaseSHA,
})
}
// and write it back
sarifBytes, err := json.Marshal(sarif) sarifBytes, err := json.Marshal(sarif)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to marshal SARIF: %v", err) return nil, fmt.Errorf("failed to marshal SARIF: %v", err)
@@ -373,14 +386,18 @@ func generateSarif(codeql CodeqlCli, nwo, databasePath, queryPackPath, databaseS
return sarifBytes, nil return sarifBytes, nil
} }
func injectVersionControlInfo(sarif *Sarif, nwo, databaseSHA string) { // XX: inlined this function
for _, run := range sarif.Runs { // func injectVersionControlInfo(sarif *Sarif, nwo, databaseSHA string) {
run.VersionControlProvenance = append(run.VersionControlProvenance, map[string]interface{}{ // // XX: is nwo name/owner or language?
"repositoryUri": fmt.Sprintf("%s/%s", os.Getenv("GITHUB_SERVER_URL"), nwo), // slog.Debug("XX: 2: is nwo a name/owner, or the original callers' queryLanguage?",
"revisionId": databaseSHA, // "nwo", nwo)
}) // for _, run := range sarif.Runs {
} // run.VersionControlProvenance = append(run.VersionControlProvenance, map[string]interface{}{
} // "repositoryUri": fmt.Sprintf("%s/%s", os.Getenv("GITHUB_SERVER_URL"), nwo),
// "revisionId": databaseSHA,
// })
// }
// }
// getSarifResultCount returns the number of results in the SARIF file. // getSarifResultCount returns the number of results in the SARIF file.
func getSarifResultCount(sarif []byte) int { func getSarifResultCount(sarif []byte) int {

View File

@@ -5,13 +5,15 @@ import (
"mrvacommander/pkg/common" "mrvacommander/pkg/common"
) )
type QueryLanguage string
// AnalyzeJob represents a job specifying a repository and a query pack to analyze it with. // AnalyzeJob represents a job specifying a repository and a query pack to analyze it with.
// This is the message format that the agent receives from the queue. // This is the message format that the agent receives from the queue.
// TODO: make query_pack_location query_pack_url with a presigned URL // TODO: make query_pack_location query_pack_url with a presigned URL
type AnalyzeJob struct { type AnalyzeJob struct {
Spec common.JobSpec // json:"job_spec" Spec common.JobSpec // json:"job_spec"
QueryPackLocation artifactstore.ArtifactLocation // json:"query_pack_location" QueryPackLocation artifactstore.ArtifactLocation // json:"query_pack_location"
QueryLanguage string // json:"query_language" QueryLanguage QueryLanguage // json:"query_language"
} }
// AnalyzeResult represents the result of an analysis job. // AnalyzeResult represents the result of an analysis job.

View File

@@ -27,7 +27,7 @@ func (c *CommanderSingle) startAnalyses(
analysisRepos *map[common.NameWithOwner]qldbstore.CodeQLDatabaseLocation, analysisRepos *map[common.NameWithOwner]qldbstore.CodeQLDatabaseLocation,
queryPackLocation artifactstore.ArtifactLocation, queryPackLocation artifactstore.ArtifactLocation,
sessionId int, sessionId int,
queryLanguage string) { queryLanguage queue.QueryLanguage) {
slog.Debug("Queueing analysis jobs", "count", len(*analysisRepos)) slog.Debug("Queueing analysis jobs", "count", len(*analysisRepos))
@@ -629,7 +629,7 @@ func (c *CommanderSingle) buildSessionInfoResponseJson(si SessionInfo) ([]byte,
Actor: actor, Actor: actor,
ControllerRepo: controllerRepo, ControllerRepo: controllerRepo,
ID: si.ID, ID: si.ID,
QueryLanguage: si.Language, QueryLanguage: string(si.Language),
QueryPackURL: si.QueryPack, QueryPackURL: si.QueryPack,
CreatedAt: time.Now().Format(time.RFC3339), CreatedAt: time.Now().Format(time.RFC3339),
UpdatedAt: time.Now().Format(time.RFC3339), UpdatedAt: time.Now().Format(time.RFC3339),
@@ -649,7 +649,7 @@ func (c *CommanderSingle) buildSessionInfoResponseJson(si SessionInfo) ([]byte,
SessionID: si.ID, SessionID: si.ID,
NameWithOwner: job.Spec.NameWithOwner, NameWithOwner: job.Spec.NameWithOwner,
}, common.JobInfo{ }, common.JobInfo{
QueryLanguage: si.Language, QueryLanguage: string(si.Language),
CreatedAt: response.CreatedAt, CreatedAt: response.CreatedAt,
UpdatedAt: response.UpdatedAt, UpdatedAt: response.UpdatedAt,
SkippedRepositories: skippedRepositories, SkippedRepositories: skippedRepositories,
@@ -667,7 +667,7 @@ func (c *CommanderSingle) buildSessionInfoResponseJson(si SessionInfo) ([]byte,
} }
func (c *CommanderSingle) collectRequestInfoAndSaveQueryPack(w http.ResponseWriter, r *http.Request, sessionId int) (string, []common.NameWithOwner, artifactstore.ArtifactLocation, error) { func (c *CommanderSingle) collectRequestInfoAndSaveQueryPack(w http.ResponseWriter, r *http.Request, sessionId int) (queue.QueryLanguage, []common.NameWithOwner, artifactstore.ArtifactLocation, error) {
slog.Debug("Collecting session info") slog.Debug("Collecting session info")
if r.Body == nil { if r.Body == nil {
@@ -708,7 +708,7 @@ func (c *CommanderSingle) collectRequestInfoAndSaveQueryPack(w http.ResponseWrit
} }
// 2. Save the language // 2. Save the language
sessionLanguage := msg.Language sessionLanguage := queue.QueryLanguage(msg.Language)
// 3. Save the repositories // 3. Save the repositories
var sessionRepos []common.NameWithOwner var sessionRepos []common.NameWithOwner

View File

@@ -11,7 +11,7 @@ import (
type SessionInfo struct { type SessionInfo struct {
ID int ID int
QueryPack string QueryPack string
Language string Language queue.QueryLanguage
AccessMismatchRepos []common.NameWithOwner AccessMismatchRepos []common.NameWithOwner
NotFoundRepos []common.NameWithOwner NotFoundRepos []common.NameWithOwner
NoCodeqlDBRepos []common.NameWithOwner NoCodeqlDBRepos []common.NameWithOwner