Go: Communicate extracted package count from extractor to autobuilder

go/extractor/cli/go-autobuilder/go-autobuilder.go

@@ -620,6 +620,16 @@ func installDependenciesAndBuild() {
 	} else {
 		log.Printf("Success: extraction succeeded for all %d discovered project(s).\n", len(workspaces))
 	}
+
+	// Check whether we have been able to extract any packages. Emit an error-level diagnostic if not.
+	// Each extractor run should have stored information about the outcome in a file in the database
+	// scratch directory, which are retrieving this information from.
+	extractionResults := make(util.ExtractionResults)
+	util.ReadExtractionResults(&extractionResults)
+
+	if extractionResults.TotalPackageCount() == 0 {
+		diagnostics.EmitGoFilesFoundButNotProcessed()
+	}
 }
 
 func main() {

go/extractor/extractor.go

@@ -124,15 +124,26 @@ func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool)
 	}
 	log.Println("Done running packages.Load.")
 
-	if len(pkgs) == 0 {
-		log.Println("No packages found.")
+	// Determine the working directory for this run of the extractor.
+	wd, err := os.Getwd()
+	if err != nil {
+		log.Printf("Warning: failed to get working directory: %s\n", err.Error())
+	} else {
+		// Determine how many packages we were able to load.
+		pkgCount := len(pkgs)
 
-		wd, err := os.Getwd()
-		if err != nil {
-			log.Printf("Warning: failed to get working directory: %s\n", err.Error())
-		} else if util.FindGoFiles(wd) {
-			diagnostics.EmitGoFilesFoundButNotProcessedForDirectory(wd)
+		if pkgCount == 0 {
+			log.Println("No packages found.")
+
+			if util.FindGoFiles(wd) {
+				diagnostics.EmitGoFilesFoundButNotProcessedForDirectory(wd)
+			}
 		}
+
+		// Write the number of packages to a file that can be inspected by the autobuilder.
+		util.WriteExtractionResult(wd, util.ExtractionResult{
+			PackageCount: pkgCount,
+		})
 	}
 
 	log.Println("Extracting universe scope.")

go/extractor/util/BUILD.bazel

@@ -5,6 +5,7 @@ load("@rules_go//go:def.bzl", "go_library", "go_test")
 go_library(
     name = "util",
     srcs = [
+        "extractor.go",
         "extractvendordirs.go",
         "semver.go",
         "util.go",

go/extractor/util/extractor.go

@@ -0,0 +1,88 @@
+package util
+
+import (
+	"encoding/json"
+	"log"
+	"os"
+	"path/filepath"
+)
+
+// Gets the path of the JSON file in the database scratch directory that is used
+// to store extraction results.
+func extractionResultsPath() string {
+	return filepath.Join(ScratchDir(), "extraction.json")
+}
+
+// Represents results of an extractor run that are of interest to the autobuilder.
+type ExtractionResult struct {
+	PackageCount int `json:"packageCount"`
+}
+
+// Represents a mapping of module roots to extraction results.
+type ExtractionResults map[string]ExtractionResult
+
+/* Returns the total number of packages extracted */
+func (results ExtractionResults) TotalPackageCount() int {
+	result := 0
+	for _, v := range results {
+		result += v.PackageCount
+	}
+	return result
+}
+
+// Reads extraction results produced by the extractor from a well-known location in the
+// database scratch directory and stores them in `results`. Returns `nil` if successful
+// or an error if not. Note that if the file does not exist, `results` are not modified
+// and `nil` is returned. If it matters whether the file was created by an extractor
+// run, then this should be checked explicitly.
+func ReadExtractionResults(results *ExtractionResults) error {
+	path := extractionResultsPath()
+
+	if FileExists(path) {
+		contents, err := os.ReadFile(path)
+
+		if err != nil {
+			log.Printf("Found %s, but could not read it: %s\n", path, err)
+			return err
+		}
+
+		if err = json.Unmarshal(contents, results); err != nil {
+			log.Printf("Failed to unmarshal JSON from %s: %s\n", path, err)
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Writes an extraction `result` for the module at root `wd` to a well-known location in the
+// database scratch directory. If the file with extraction results exists already, it is updated.
+// Note: this assumes that multiple copies of the extractor are not run concurrently.
+func WriteExtractionResult(wd string, result ExtractionResult) {
+	path := extractionResultsPath()
+	results := make(ExtractionResults)
+
+	// Create the scratch directory, if needed.
+	if !DirExists(ScratchDir()) {
+		os.Mkdir(ScratchDir(), 0755)
+	}
+
+	// Read existing extraction results, if there are any.
+	ReadExtractionResults(&results)
+
+	// Store the new extraction result.
+	results[wd] = result
+
+	// Write all results to the file as JSON.
+	bytes, err := json.Marshal(results)
+
+	if err != nil {
+		log.Printf("Unable to marshal: %s\n", err)
+	}
+
+	err = os.WriteFile(path, bytes, 0644)
+
+	if err != nil {
+		log.Printf("Failed to write to %s: %s\n", path, err)
+	}
+}

go/extractor/util/util.go

@@ -34,6 +34,11 @@ func Getenv(key string, aliases ...string) string {
 	return ""
 }
 
+// Retrieves the path of the scratch directory in the database.
+func ScratchDir() string {
+	return os.Getenv("CODEQL_EXTRACTOR_GO_SCRATCH_DIR")
+}
+
 // FileExists tests whether the file at `filename` exists and is not a directory.
 func FileExists(filename string) bool {
 	info, err := os.Stat(filename)