diff --git a/go/extractor/cli/go-autobuilder/go-autobuilder.go b/go/extractor/cli/go-autobuilder/go-autobuilder.go index c22b356c4be..756bd19b45e 100644 --- a/go/extractor/cli/go-autobuilder/go-autobuilder.go +++ b/go/extractor/cli/go-autobuilder/go-autobuilder.go @@ -448,7 +448,7 @@ func installDependencies(workspace project.GoWorkspace) { } // Run the extractor. -func extract(workspace project.GoWorkspace) bool { +func extract(workspace project.GoWorkspace, sourceRoot string) bool { extractor, err := util.GetExtractorPath() if err != nil { log.Fatalf("Could not determine path of extractor: %v.\n", err) @@ -459,6 +459,12 @@ func extract(workspace project.GoWorkspace) bool { extractorArgs = append(extractorArgs, workspace.ModMode.ArgsForGoVersion(toolchain.GetEnvGoSemVer())...) } + if util.IsOverlayExtraction() { + // When we are extracting an overlay, pass the source root to the extractor so that it knows + // how to resolve the relative paths in the list of changed files. + extractorArgs = append(extractorArgs, "--source-root", sourceRoot) + } + if len(workspace.Modules) == 0 { // There may be no modules if we are using e.g. Dep or Glide extractorArgs = append(extractorArgs, "./...") @@ -587,6 +593,12 @@ func installDependenciesAndBuild() { buildWithCustomCommands(inst) } + // The autobuilder is invoked with its working directory set to the source directory. + sourceRoot, err := os.Getwd() + if err != nil { + log.Fatalf("Failed to get current working directory: %s\n", err.Error()) + } + // Attempt to extract all workspaces; we will tolerate individual extraction failures here for i, workspace := range workspaces { if workspace.ModMode == project.ModVendor { @@ -607,7 +619,7 @@ func installDependenciesAndBuild() { } } - workspaces[i].Extracted = extract(workspace) + workspaces[i].Extracted = extract(workspace, sourceRoot) if !workspaces[i].Extracted { unsuccessfulProjects = append(unsuccessfulProjects, workspace.BaseDir) @@ -632,6 +644,8 @@ func installDependenciesAndBuild() { } else { log.Printf("Success: extraction succeeded for all %d discovered project(s).\n", len(workspaces)) } + + util.WriteOverlayBaseMetadata() } func main() { diff --git a/go/extractor/cli/go-extractor/go-extractor.go b/go/extractor/cli/go-extractor/go-extractor.go index 72ab3291f3c..d394d5fde40 100644 --- a/go/extractor/cli/go-extractor/go-extractor.go +++ b/go/extractor/cli/go-extractor/go-extractor.go @@ -24,9 +24,10 @@ func usage() { // extractTests is set (a) if we were manually commanded to extract tests via the relevant // environment variable / extractor option, or (b) we're mimicking a `go test` command. -func parseFlags(args []string, mimic bool, extractTests bool) ([]string, []string, bool) { +func parseFlags(args []string, mimic bool, extractTests bool) ([]string, []string, bool, string) { i := 0 buildFlags := []string{} + var sourceRoot string for ; i < len(args) && strings.HasPrefix(args[i], "-"); i++ { if args[i] == "--" { i++ @@ -61,6 +62,18 @@ func parseFlags(args []string, mimic bool, extractTests bool) ([]string, []strin } else { log.Fatalf("--mimic requires an argument, e.g. --mimic go") } + case "--source-root": + // The extractor can be called by the autobuilder with the working directory set to + // the directory containing the workspace we're extracting, and this may be a + // subdirectory of the actual source root. This argument lets us resolve paths that + // are relative to that source root, e.g. for the list of overlay changed files. + if i+1 < len(args) { + i++ + sourceRoot = args[i] + log.Printf("Source root is %s", sourceRoot) + } else { + log.Fatalf("--source-root requires an argument, e.g. --source-root /path/to/root") + } } } @@ -93,14 +106,14 @@ func parseFlags(args []string, mimic bool, extractTests bool) ([]string, []strin cpuprofile = os.Getenv("CODEQL_EXTRACTOR_GO_CPU_PROFILE") memprofile = os.Getenv("CODEQL_EXTRACTOR_GO_MEM_PROFILE") - return buildFlags, args[i:], extractTests + return buildFlags, args[i:], extractTests, sourceRoot } func main() { util.SetLogLevel() extractTestsDefault := os.Getenv("CODEQL_EXTRACTOR_GO_OPTION_EXTRACT_TESTS") == "true" - buildFlags, patterns, extractTests := parseFlags(os.Args[1:], false, extractTestsDefault) + buildFlags, patterns, extractTests, sourceRoot := parseFlags(os.Args[1:], false, extractTestsDefault) if cpuprofile != "" { f, err := os.Create(cpuprofile) @@ -120,7 +133,7 @@ func main() { } log.Printf("Build flags: '%s'; patterns: '%s'\n", strings.Join(buildFlags, " "), strings.Join(patterns, " ")) - err := extractor.ExtractWithFlags(buildFlags, patterns, extractTests) + err := extractor.ExtractWithFlags(buildFlags, patterns, extractTests, sourceRoot) if err != nil { errString := err.Error() if strings.Contains(errString, "unexpected directory layout:") { diff --git a/go/extractor/extractor.go b/go/extractor/extractor.go index b4d4815f15c..0ddd812cc7a 100644 --- a/go/extractor/extractor.go +++ b/go/extractor/extractor.go @@ -58,16 +58,11 @@ func init() { } } -// Extract extracts the packages specified by the given patterns -func Extract(patterns []string) error { - return ExtractWithFlags(nil, patterns, false) -} - // ExtractWithFlags extracts the packages specified by the given patterns and build flags -func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool) error { +func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool, sourceRoot string) error { startTime := time.Now() - extraction := NewExtraction(buildFlags, patterns) + extraction := NewExtraction(buildFlags, patterns, sourceRoot) defer extraction.StatWriter.Close() modEnabled := os.Getenv("GO111MODULE") != "off" @@ -311,8 +306,6 @@ func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool) extraction.WaitGroup.Wait() - util.WriteOverlayBaseMetadata() - log.Println("Done extracting packages.") t := time.Now() @@ -370,7 +363,7 @@ func (extraction *Extraction) GetNextErr(path string) int { return res } -func NewExtraction(buildFlags []string, patterns []string) *Extraction { +func NewExtraction(buildFlags []string, patterns []string, sourceRoot string) *Extraction { hash := md5.New() io.WriteString(hash, "go") for _, buildFlag := range buildFlags { @@ -382,7 +375,7 @@ func NewExtraction(buildFlags []string, patterns []string) *Extraction { } sum := hash.Sum(nil) - overlayChangeList := util.GetOverlayChanges() + overlayChangeList := util.GetOverlayChanges(sourceRoot) var overlayChanges map[string]bool if overlayChangeList == nil { overlayChanges = nil diff --git a/go/extractor/util/overlays.go b/go/extractor/util/overlays.go index 07fad8b4d71..6a5c3760549 100644 --- a/go/extractor/util/overlays.go +++ b/go/extractor/util/overlays.go @@ -4,12 +4,18 @@ import ( "encoding/json" "log" "os" + "path/filepath" ) +func IsOverlayExtraction() bool { + _, present := os.LookupEnv("CODEQL_EXTRACTOR_GO_OVERLAY_METADATA_IN") + return present +} + // If the relevant environment variable is set, indicating that we are extracting an overlay // database, GetOverlayChanges returns the list of relative paths of files that have changed (or // been deleted). Otherwise, it returns `nil`. -func GetOverlayChanges() []string { +func GetOverlayChanges(sourceRoot string) []string { if overlayChangesJsonPath, present := os.LookupEnv("CODEQL_EXTRACTOR_GO_OVERLAY_CHANGES"); present { log.Printf("Reading overlay changes from: %s", overlayChangesJsonPath) @@ -28,7 +34,19 @@ func GetOverlayChanges() []string { log.Fatalf("Failed to decode overlay changes JSON file: %s", err) } - return overlayData.Changes + absPaths := make([]string, 0, len(overlayData.Changes)) + if sourceRoot == "" { + // This shouldn't happen, because it implies the extractor was invoked in some way other + // than from the autobuilder. However, we'll only attempt to build an overlay if there + // exists an overlay _base_, and only the autobuilder writes the metadata file that + // ensures a database is created as an overlay-base. + log.Fatalf("Extractor is running in overlay mode, but --source-root was not provided") + } + for _, relPath := range overlayData.Changes { + absPaths = append(absPaths, filepath.Clean(sourceRoot+"/"+relPath)) + } + + return absPaths } else { return nil }