From 8532605be701c550e7e99412ca345e98603a8a73 Mon Sep 17 00:00:00 2001 From: xhd2015 Date: Sun, 25 Jul 2021 14:44:04 +0800 Subject: [PATCH] Accelerating go-extractor by using 'go list -deps' instead of just 'go list' Change-Id: Icc77214809a0bb8536d751f21194690d58663dc5 --- extractor/extractor.go | 31 ++++++++------ extractor/util/util.go | 97 +++++++++++++++++++++++++++--------------- 2 files changed, 81 insertions(+), 47 deletions(-) diff --git a/extractor/extractor.go b/extractor/extractor.go index 4208a788d02..e9a30f08c7d 100644 --- a/extractor/extractor.go +++ b/extractor/extractor.go @@ -109,6 +109,25 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error { // root directories of packages that we want to extract wantedRoots := make(map[string]bool) + log.Printf("Running go list to resolve package and module directories.") + // get all packages information + pkgsInfo, err := util.GetPkgsInfo(patterns, true, modFlags...) + if err != nil { + log.Fatalf("Error getting dependency package or module directories: %v.", err) + } + for pkgPath, pkgInfo := range pkgsInfo { + mdir := pkgInfo.ModDir + pdir := pkgInfo.PkgDir + // GetPkgsInfo returns the empty string if the module directory cannot be determined, e.g. if the package + // is not using modules. If this is the case, fall back to the package directory + if mdir == "" { + mdir = pdir + } + pkgRoots[pkgPath] = mdir + pkgDirs[pkgPath] = pdir + } + log.Printf("Done running go list deps: resolved %d packages.", len(pkgsInfo)) + // recursively visit all packages in depth-first order; // on the way down, associate each package scope with its corresponding package, // and on the way up extract the package's scope @@ -117,18 +136,6 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error { }, func(pkg *packages.Package) { log.Printf("Processing package %s.", pkg.PkgPath) - if _, ok := pkgRoots[pkg.PkgPath]; !ok { - mdir := util.GetModDir(pkg.PkgPath, modFlags...) - pdir := util.GetPkgDir(pkg.PkgPath, modFlags...) - // GetModDir returns the empty string if the module directory cannot be determined, e.g. if the package - // is not using modules. If this is the case, fall back to the package directory - if mdir == "" { - mdir = pdir - } - pkgRoots[pkg.PkgPath] = mdir - pkgDirs[pkg.PkgPath] = pdir - } - log.Printf("Extracting types for package %s.", pkg.PkgPath) tw, err := trap.NewWriter(pkg.PkgPath, pkg) diff --git a/extractor/util/util.go b/extractor/util/util.go index 5725c03d5b6..58d488d4e74 100644 --- a/extractor/util/util.go +++ b/extractor/util/util.go @@ -1,7 +1,9 @@ package util import ( + "encoding/json" "errors" + "io" "log" "os" "os/exec" @@ -32,14 +34,17 @@ func Getenv(key string, aliases ...string) string { // runGoList is a helper function for running go list with format `format` and flags `flags` on // package `pkgpath`. func runGoList(format string, pkgpath string, flags ...string) (string, error) { - return runGoListWithEnv(format, pkgpath, nil, flags...) + return runGoListWithEnv(format, []string{pkgpath}, nil, flags...) } -func runGoListWithEnv(format string, pkgpath string, additionalEnv []string, flags ...string) (string, error) { +// runGoListWithEnv is a helper function for running go list with format `format` and flags `flags` on +// pattern `patterns`, with environment variables `additionalEnv`, which is a slice of strings in the format `NAME=VALUE`. +func runGoListWithEnv(format string, patterns []string, additionalEnv []string, flags ...string) (string, error) { args := append([]string{"list", "-e", "-f", format}, flags...) - args = append(args, pkgpath) + args = append(args, patterns...) cmd := exec.Command("go", args...) cmd.Env = append(os.Environ(), additionalEnv...) + log.Printf("Running cmd: %s", cmd.String()) out, err := cmd.Output() if err != nil { @@ -54,44 +59,66 @@ func runGoListWithEnv(format string, pkgpath string, additionalEnv []string, fla return strings.TrimSpace(string(out)), nil } -// GetModDir gets the absolute directory of the module containing the package with path -// `pkgpath`. It passes the `go list` the flags specified by `flags`. -func GetModDir(pkgpath string, flags ...string) string { - // enable module mode so that we can find a module root if it exists, even if go module support is - // disabled by a build - mod, err := runGoListWithEnv("{{.Module}}", pkgpath, []string{"GO111MODULE=on"}, flags...) - if err != nil || mod == "" { - // if the command errors or modules aren't being used, return the empty string - return "" - } - - modDir, err := runGoListWithEnv("{{.Module.Dir}}", pkgpath, []string{"GO111MODULE=on"}, flags...) - if err != nil { - return "" - } - - abs, err := filepath.Abs(modDir) - if err != nil { - log.Printf("Warning: unable to make %s absolute: %s", modDir, err.Error()) - return "" - } - return abs +// PkgInfo holds package directory and module directory(if any) for a package +type PkgInfo struct { + PkgDir string // the directory directly containing source code of this pacakge + ModDir string // the module directory containing this package, empty if not a module } -// GetPkgDir gets the absolute directory containing the package with path `pkgpath`. It passes the -// `go list` command the flags specified by `flags`. -func GetPkgDir(pkgpath string, flags ...string) string { - pkgDir, err := runGoList("{{.Dir}}", pkgpath, flags...) - if err != nil { - return "" +// GetPkgsInfo gets absolute directories of module and package root, where packages are denoted by `patterns`. +// It passes the `go list` the flags specified by `flags`. +// If `includingDeps`, all dependencies will also be included. see `go help list` for `-deps` flag +func GetPkgsInfo(patterns []string, includingDeps bool, flags ...string) (map[string]*PkgInfo, error) { + // enable module mode so that we can find a module root if it exists, even if go module support is + // disabled by a build + if includingDeps { + // the flag `-deps` causes all dependencies to be retrieved + flags = append(flags, "-deps") } - abs, err := filepath.Abs(pkgDir) + // using -json overrides -f format + output, err := runGoListWithEnv("", patterns, []string{"GO111MODULE=on"}, append(flags, "-json")...) if err != nil { - log.Printf("Warning: unable to make %s absolute: %s", pkgDir, err.Error()) - return "" + return nil, err } - return abs + + // the output of `go list -json` is a stream of json object + type goListPkgInfo struct { + ImportPath string + Dir string + Module *struct { + Dir string + } + } + pkgInfoMapping := make(map[string]*PkgInfo) + streamDecoder := json.NewDecoder(strings.NewReader(output)) + for { + var pkgInfo goListPkgInfo + decErr := streamDecoder.Decode(&pkgInfo) + if decErr == io.EOF { + break + } + if decErr != nil { + log.Printf("Error decoding output of go list -json: %s", err.Error()) + return nil, decErr + } + pkgAbsDir, err := filepath.Abs(pkgInfo.Dir) + if err != nil { + log.Printf("Unable to make package dir %s absolute: %s", pkgInfo.Dir, err.Error()) + } + var modAbsDir string + if pkgInfo.Module != nil { + modAbsDir, err = filepath.Abs(pkgInfo.Module.Dir) + if err != nil { + log.Printf("Unable to make module dir %s absolute: %s", pkgInfo.Module.Dir, err.Error()) + } + } + pkgInfoMapping[pkgInfo.ImportPath] = &PkgInfo{ + PkgDir: pkgAbsDir, + ModDir: modAbsDir, + } + } + return pkgInfoMapping, nil } // DepErrors checks there are any errors resolving dependencies for `pkgpath`. It passes the `go