From 1f8e82148d424a69611d5ea06620ffdaee8713ee Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Mon, 7 Oct 2024 12:54:50 +0100 Subject: [PATCH] Go: Communicate extracted package count from extractor to autobuilder --- .../cli/go-autobuilder/go-autobuilder.go | 10 +++ go/extractor/extractor.go | 25 ++++-- go/extractor/util/BUILD.bazel | 1 + go/extractor/util/extractor.go | 88 +++++++++++++++++++ go/extractor/util/util.go | 5 ++ 5 files changed, 122 insertions(+), 7 deletions(-) create mode 100644 go/extractor/util/extractor.go diff --git a/go/extractor/cli/go-autobuilder/go-autobuilder.go b/go/extractor/cli/go-autobuilder/go-autobuilder.go index aeff4f0bd6e..9c9c10bf0df 100644 --- a/go/extractor/cli/go-autobuilder/go-autobuilder.go +++ b/go/extractor/cli/go-autobuilder/go-autobuilder.go @@ -620,6 +620,16 @@ func installDependenciesAndBuild() { } else { log.Printf("Success: extraction succeeded for all %d discovered project(s).\n", len(workspaces)) } + + // Check whether we have been able to extract any packages. Emit an error-level diagnostic if not. + // Each extractor run should have stored information about the outcome in a file in the database + // scratch directory, which are retrieving this information from. + extractionResults := make(util.ExtractionResults) + util.ReadExtractionResults(&extractionResults) + + if extractionResults.TotalPackageCount() == 0 { + diagnostics.EmitGoFilesFoundButNotProcessed() + } } func main() { diff --git a/go/extractor/extractor.go b/go/extractor/extractor.go index 55e293f9900..fc85a9918b2 100644 --- a/go/extractor/extractor.go +++ b/go/extractor/extractor.go @@ -124,15 +124,26 @@ func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool) } log.Println("Done running packages.Load.") - if len(pkgs) == 0 { - log.Println("No packages found.") + // Determine the working directory for this run of the extractor. + wd, err := os.Getwd() + if err != nil { + log.Printf("Warning: failed to get working directory: %s\n", err.Error()) + } else { + // Determine how many packages we were able to load. + pkgCount := len(pkgs) - wd, err := os.Getwd() - if err != nil { - log.Printf("Warning: failed to get working directory: %s\n", err.Error()) - } else if util.FindGoFiles(wd) { - diagnostics.EmitGoFilesFoundButNotProcessedForDirectory(wd) + if pkgCount == 0 { + log.Println("No packages found.") + + if util.FindGoFiles(wd) { + diagnostics.EmitGoFilesFoundButNotProcessedForDirectory(wd) + } } + + // Write the number of packages to a file that can be inspected by the autobuilder. + util.WriteExtractionResult(wd, util.ExtractionResult{ + PackageCount: pkgCount, + }) } log.Println("Extracting universe scope.") diff --git a/go/extractor/util/BUILD.bazel b/go/extractor/util/BUILD.bazel index b7a7783aa79..50bf7e38f78 100644 --- a/go/extractor/util/BUILD.bazel +++ b/go/extractor/util/BUILD.bazel @@ -5,6 +5,7 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "util", srcs = [ + "extractor.go", "extractvendordirs.go", "semver.go", "util.go", diff --git a/go/extractor/util/extractor.go b/go/extractor/util/extractor.go new file mode 100644 index 00000000000..c700c2cd2da --- /dev/null +++ b/go/extractor/util/extractor.go @@ -0,0 +1,88 @@ +package util + +import ( + "encoding/json" + "log" + "os" + "path/filepath" +) + +// Gets the path of the JSON file in the database scratch directory that is used +// to store extraction results. +func extractionResultsPath() string { + return filepath.Join(ScratchDir(), "extraction.json") +} + +// Represents results of an extractor run that are of interest to the autobuilder. +type ExtractionResult struct { + PackageCount int `json:"packageCount"` +} + +// Represents a mapping of module roots to extraction results. +type ExtractionResults map[string]ExtractionResult + +/* Returns the total number of packages extracted */ +func (results ExtractionResults) TotalPackageCount() int { + result := 0 + for _, v := range results { + result += v.PackageCount + } + return result +} + +// Reads extraction results produced by the extractor from a well-known location in the +// database scratch directory and stores them in `results`. Returns `nil` if successful +// or an error if not. Note that if the file does not exist, `results` are not modified +// and `nil` is returned. If it matters whether the file was created by an extractor +// run, then this should be checked explicitly. +func ReadExtractionResults(results *ExtractionResults) error { + path := extractionResultsPath() + + if FileExists(path) { + contents, err := os.ReadFile(path) + + if err != nil { + log.Printf("Found %s, but could not read it: %s\n", path, err) + return err + } + + if err = json.Unmarshal(contents, results); err != nil { + log.Printf("Failed to unmarshal JSON from %s: %s\n", path, err) + return err + } + } + + return nil +} + +// Writes an extraction `result` for the module at root `wd` to a well-known location in the +// database scratch directory. If the file with extraction results exists already, it is updated. +// Note: this assumes that multiple copies of the extractor are not run concurrently. +func WriteExtractionResult(wd string, result ExtractionResult) { + path := extractionResultsPath() + results := make(ExtractionResults) + + // Create the scratch directory, if needed. + if !DirExists(ScratchDir()) { + os.Mkdir(ScratchDir(), 0755) + } + + // Read existing extraction results, if there are any. + ReadExtractionResults(&results) + + // Store the new extraction result. + results[wd] = result + + // Write all results to the file as JSON. + bytes, err := json.Marshal(results) + + if err != nil { + log.Printf("Unable to marshal: %s\n", err) + } + + err = os.WriteFile(path, bytes, 0644) + + if err != nil { + log.Printf("Failed to write to %s: %s\n", path, err) + } +} diff --git a/go/extractor/util/util.go b/go/extractor/util/util.go index 595326496d8..51c434d7dbc 100644 --- a/go/extractor/util/util.go +++ b/go/extractor/util/util.go @@ -34,6 +34,11 @@ func Getenv(key string, aliases ...string) string { return "" } +// Retrieves the path of the scratch directory in the database. +func ScratchDir() string { + return os.Getenv("CODEQL_EXTRACTOR_GO_SCRATCH_DIR") +} + // FileExists tests whether the file at `filename` exists and is not a directory. func FileExists(filename string) bool { info, err := os.Stat(filename)