From 8532605be701c550e7e99412ca345e98603a8a73 Mon Sep 17 00:00:00 2001 From: xhd2015 Date: Sun, 25 Jul 2021 14:44:04 +0800 Subject: [PATCH 1/5] Accelerating go-extractor by using 'go list -deps' instead of just 'go list' Change-Id: Icc77214809a0bb8536d751f21194690d58663dc5 --- extractor/extractor.go | 31 ++++++++------ extractor/util/util.go | 97 +++++++++++++++++++++++++++--------------- 2 files changed, 81 insertions(+), 47 deletions(-) diff --git a/extractor/extractor.go b/extractor/extractor.go index 4208a788d02..e9a30f08c7d 100644 --- a/extractor/extractor.go +++ b/extractor/extractor.go @@ -109,6 +109,25 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error { // root directories of packages that we want to extract wantedRoots := make(map[string]bool) + log.Printf("Running go list to resolve package and module directories.") + // get all packages information + pkgsInfo, err := util.GetPkgsInfo(patterns, true, modFlags...) + if err != nil { + log.Fatalf("Error getting dependency package or module directories: %v.", err) + } + for pkgPath, pkgInfo := range pkgsInfo { + mdir := pkgInfo.ModDir + pdir := pkgInfo.PkgDir + // GetPkgsInfo returns the empty string if the module directory cannot be determined, e.g. if the package + // is not using modules. If this is the case, fall back to the package directory + if mdir == "" { + mdir = pdir + } + pkgRoots[pkgPath] = mdir + pkgDirs[pkgPath] = pdir + } + log.Printf("Done running go list deps: resolved %d packages.", len(pkgsInfo)) + // recursively visit all packages in depth-first order; // on the way down, associate each package scope with its corresponding package, // and on the way up extract the package's scope @@ -117,18 +136,6 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error { }, func(pkg *packages.Package) { log.Printf("Processing package %s.", pkg.PkgPath) - if _, ok := pkgRoots[pkg.PkgPath]; !ok { - mdir := util.GetModDir(pkg.PkgPath, modFlags...) - pdir := util.GetPkgDir(pkg.PkgPath, modFlags...) - // GetModDir returns the empty string if the module directory cannot be determined, e.g. if the package - // is not using modules. If this is the case, fall back to the package directory - if mdir == "" { - mdir = pdir - } - pkgRoots[pkg.PkgPath] = mdir - pkgDirs[pkg.PkgPath] = pdir - } - log.Printf("Extracting types for package %s.", pkg.PkgPath) tw, err := trap.NewWriter(pkg.PkgPath, pkg) diff --git a/extractor/util/util.go b/extractor/util/util.go index 5725c03d5b6..58d488d4e74 100644 --- a/extractor/util/util.go +++ b/extractor/util/util.go @@ -1,7 +1,9 @@ package util import ( + "encoding/json" "errors" + "io" "log" "os" "os/exec" @@ -32,14 +34,17 @@ func Getenv(key string, aliases ...string) string { // runGoList is a helper function for running go list with format `format` and flags `flags` on // package `pkgpath`. func runGoList(format string, pkgpath string, flags ...string) (string, error) { - return runGoListWithEnv(format, pkgpath, nil, flags...) + return runGoListWithEnv(format, []string{pkgpath}, nil, flags...) } -func runGoListWithEnv(format string, pkgpath string, additionalEnv []string, flags ...string) (string, error) { +// runGoListWithEnv is a helper function for running go list with format `format` and flags `flags` on +// pattern `patterns`, with environment variables `additionalEnv`, which is a slice of strings in the format `NAME=VALUE`. +func runGoListWithEnv(format string, patterns []string, additionalEnv []string, flags ...string) (string, error) { args := append([]string{"list", "-e", "-f", format}, flags...) - args = append(args, pkgpath) + args = append(args, patterns...) cmd := exec.Command("go", args...) cmd.Env = append(os.Environ(), additionalEnv...) + log.Printf("Running cmd: %s", cmd.String()) out, err := cmd.Output() if err != nil { @@ -54,44 +59,66 @@ func runGoListWithEnv(format string, pkgpath string, additionalEnv []string, fla return strings.TrimSpace(string(out)), nil } -// GetModDir gets the absolute directory of the module containing the package with path -// `pkgpath`. It passes the `go list` the flags specified by `flags`. -func GetModDir(pkgpath string, flags ...string) string { - // enable module mode so that we can find a module root if it exists, even if go module support is - // disabled by a build - mod, err := runGoListWithEnv("{{.Module}}", pkgpath, []string{"GO111MODULE=on"}, flags...) - if err != nil || mod == "" { - // if the command errors or modules aren't being used, return the empty string - return "" - } - - modDir, err := runGoListWithEnv("{{.Module.Dir}}", pkgpath, []string{"GO111MODULE=on"}, flags...) - if err != nil { - return "" - } - - abs, err := filepath.Abs(modDir) - if err != nil { - log.Printf("Warning: unable to make %s absolute: %s", modDir, err.Error()) - return "" - } - return abs +// PkgInfo holds package directory and module directory(if any) for a package +type PkgInfo struct { + PkgDir string // the directory directly containing source code of this pacakge + ModDir string // the module directory containing this package, empty if not a module } -// GetPkgDir gets the absolute directory containing the package with path `pkgpath`. It passes the -// `go list` command the flags specified by `flags`. -func GetPkgDir(pkgpath string, flags ...string) string { - pkgDir, err := runGoList("{{.Dir}}", pkgpath, flags...) - if err != nil { - return "" +// GetPkgsInfo gets absolute directories of module and package root, where packages are denoted by `patterns`. +// It passes the `go list` the flags specified by `flags`. +// If `includingDeps`, all dependencies will also be included. see `go help list` for `-deps` flag +func GetPkgsInfo(patterns []string, includingDeps bool, flags ...string) (map[string]*PkgInfo, error) { + // enable module mode so that we can find a module root if it exists, even if go module support is + // disabled by a build + if includingDeps { + // the flag `-deps` causes all dependencies to be retrieved + flags = append(flags, "-deps") } - abs, err := filepath.Abs(pkgDir) + // using -json overrides -f format + output, err := runGoListWithEnv("", patterns, []string{"GO111MODULE=on"}, append(flags, "-json")...) if err != nil { - log.Printf("Warning: unable to make %s absolute: %s", pkgDir, err.Error()) - return "" + return nil, err } - return abs + + // the output of `go list -json` is a stream of json object + type goListPkgInfo struct { + ImportPath string + Dir string + Module *struct { + Dir string + } + } + pkgInfoMapping := make(map[string]*PkgInfo) + streamDecoder := json.NewDecoder(strings.NewReader(output)) + for { + var pkgInfo goListPkgInfo + decErr := streamDecoder.Decode(&pkgInfo) + if decErr == io.EOF { + break + } + if decErr != nil { + log.Printf("Error decoding output of go list -json: %s", err.Error()) + return nil, decErr + } + pkgAbsDir, err := filepath.Abs(pkgInfo.Dir) + if err != nil { + log.Printf("Unable to make package dir %s absolute: %s", pkgInfo.Dir, err.Error()) + } + var modAbsDir string + if pkgInfo.Module != nil { + modAbsDir, err = filepath.Abs(pkgInfo.Module.Dir) + if err != nil { + log.Printf("Unable to make module dir %s absolute: %s", pkgInfo.Module.Dir, err.Error()) + } + } + pkgInfoMapping[pkgInfo.ImportPath] = &PkgInfo{ + PkgDir: pkgAbsDir, + ModDir: modAbsDir, + } + } + return pkgInfoMapping, nil } // DepErrors checks there are any errors resolving dependencies for `pkgpath`. It passes the `go From 6ed61939734f5e16411541bbc1205cbdc09114f9 Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Wed, 11 Aug 2021 23:33:13 -0700 Subject: [PATCH 2/5] Remove redundant map assignments and fix some typos --- extractor/extractor.go | 38 +++++++++++++++----------------------- extractor/util/util.go | 15 ++++++++------- 2 files changed, 23 insertions(+), 30 deletions(-) diff --git a/extractor/extractor.go b/extractor/extractor.go index e9a30f08c7d..50bf9d78c96 100644 --- a/extractor/extractor.go +++ b/extractor/extractor.go @@ -102,31 +102,18 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error { extractUniverseScope() log.Println("Done extracting universe scope.") - // a map of package path to package root directory (currently the module root or the source directory) - pkgRoots := make(map[string]string) - // a map of package path to source code directory - pkgDirs := make(map[string]string) + // a map of package path to source directory and module root directory + pkgInfos := make(map[string]*util.PkgInfo) // root directories of packages that we want to extract wantedRoots := make(map[string]bool) log.Printf("Running go list to resolve package and module directories.") // get all packages information - pkgsInfo, err := util.GetPkgsInfo(patterns, true, modFlags...) + pkgInfos, err = util.GetPkgsInfo(patterns, true, modFlags...) if err != nil { log.Fatalf("Error getting dependency package or module directories: %v.", err) } - for pkgPath, pkgInfo := range pkgsInfo { - mdir := pkgInfo.ModDir - pdir := pkgInfo.PkgDir - // GetPkgsInfo returns the empty string if the module directory cannot be determined, e.g. if the package - // is not using modules. If this is the case, fall back to the package directory - if mdir == "" { - mdir = pdir - } - pkgRoots[pkgPath] = mdir - pkgDirs[pkgPath] = pdir - } - log.Printf("Done running go list deps: resolved %d packages.", len(pkgsInfo)) + log.Printf("Done running go list deps: resolved %d packages.", len(pkgInfos)) // recursively visit all packages in depth-first order; // on the way down, associate each package scope with its corresponding package, @@ -160,11 +147,14 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error { }) for _, pkg := range pkgs { - if pkgRoots[pkg.PkgPath] == "" { + info := pkgInfos[pkg.PkgPath] + if info.PkgDir == "" { log.Fatalf("Unable to get a source directory for input package %s.", pkg.PkgPath) } - wantedRoots[pkgRoots[pkg.PkgPath]] = true - wantedRoots[pkgDirs[pkg.PkgPath]] = true + wantedRoots[info.PkgDir] = true + if info.ModDir != "" { + wantedRoots[info.ModDir] = true + } } log.Println("Done processing dependencies.") @@ -182,7 +172,9 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error { return true }, func(pkg *packages.Package) { for root, _ := range wantedRoots { - relDir, err := filepath.Rel(root, pkgDirs[pkg.PkgPath]) + info := pkgInfos[pkg.PkgPath] + + relDir, err := filepath.Rel(root, info.PkgDir) if err != nil || noExtractRe.MatchString(relDir) { // if the path can't be made relative or matches the noExtract regexp skip it continue @@ -190,8 +182,8 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error { extraction.extractPackage(pkg) - if pkgRoots[pkg.PkgPath] != "" { - modPath := filepath.Join(pkgRoots[pkg.PkgPath], "go.mod") + if info.ModDir != "" { + modPath := filepath.Join(info.ModDir, "go.mod") if util.FileExists(modPath) { log.Printf("Extracting %s", modPath) start := time.Now() diff --git a/extractor/util/util.go b/extractor/util/util.go index 58d488d4e74..9ec9a8c5212 100644 --- a/extractor/util/util.go +++ b/extractor/util/util.go @@ -14,9 +14,9 @@ import ( var extractorPath string -// Getenv retrieves the value of the environment variable named by the key. -// If that variable is not present, it iterates over the given aliases until -// it finds one that is. If none are present, the empty string is returned. +// Getenv retrieves the value of the environment variable named by the key. If that variable is not +// present, it iterates over the given aliases until it finds one that is. If none are present, the +// empty string is returned. func Getenv(key string, aliases ...string) string { val := os.Getenv(key) if val != "" { @@ -37,8 +37,9 @@ func runGoList(format string, pkgpath string, flags ...string) (string, error) { return runGoListWithEnv(format, []string{pkgpath}, nil, flags...) } -// runGoListWithEnv is a helper function for running go list with format `format` and flags `flags` on -// pattern `patterns`, with environment variables `additionalEnv`, which is a slice of strings in the format `NAME=VALUE`. +// runGoListWithEnv is a helper function for running go list with format `format` and flags `flags` +// on pattern `patterns`, with environment variables `additionalEnv`, which is a slice of strings in +// the format `NAME=VALUE`. func runGoListWithEnv(format string, patterns []string, additionalEnv []string, flags ...string) (string, error) { args := append([]string{"list", "-e", "-f", format}, flags...) args = append(args, patterns...) @@ -59,9 +60,9 @@ func runGoListWithEnv(format string, patterns []string, additionalEnv []string, return strings.TrimSpace(string(out)), nil } -// PkgInfo holds package directory and module directory(if any) for a package +// PkgInfo holds package directory and module directory (if any) for a package type PkgInfo struct { - PkgDir string // the directory directly containing source code of this pacakge + PkgDir string // the directory directly containing source code of this package ModDir string // the module directory containing this package, empty if not a module } From 89c9c7060cf6f25b27e5d0ceb5c0d60260b0306b Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Wed, 25 Aug 2021 12:41:42 -0700 Subject: [PATCH 3/5] Remove unnecessary environment set --- extractor/util/util.go | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/extractor/util/util.go b/extractor/util/util.go index 9ec9a8c5212..24aebbad5e1 100644 --- a/extractor/util/util.go +++ b/extractor/util/util.go @@ -32,19 +32,11 @@ func Getenv(key string, aliases ...string) string { } // runGoList is a helper function for running go list with format `format` and flags `flags` on -// package `pkgpath`. -func runGoList(format string, pkgpath string, flags ...string) (string, error) { - return runGoListWithEnv(format, []string{pkgpath}, nil, flags...) -} - -// runGoListWithEnv is a helper function for running go list with format `format` and flags `flags` -// on pattern `patterns`, with environment variables `additionalEnv`, which is a slice of strings in -// the format `NAME=VALUE`. -func runGoListWithEnv(format string, patterns []string, additionalEnv []string, flags ...string) (string, error) { +// the packages defined by `patterns`. +func runGoList(format string, patterns []string, flags ...string) (string, error) { args := append([]string{"list", "-e", "-f", format}, flags...) args = append(args, patterns...) cmd := exec.Command("go", args...) - cmd.Env = append(os.Environ(), additionalEnv...) log.Printf("Running cmd: %s", cmd.String()) out, err := cmd.Output() @@ -78,7 +70,7 @@ func GetPkgsInfo(patterns []string, includingDeps bool, flags ...string) (map[st } // using -json overrides -f format - output, err := runGoListWithEnv("", patterns, []string{"GO111MODULE=on"}, append(flags, "-json")...) + output, err := runGoList("", patterns, append(flags, "-json")...) if err != nil { return nil, err } @@ -125,7 +117,7 @@ func GetPkgsInfo(patterns []string, includingDeps bool, flags ...string) (map[st // DepErrors checks there are any errors resolving dependencies for `pkgpath`. It passes the `go // list` command the flags specified by `flags`. func DepErrors(pkgpath string, flags ...string) bool { - out, err := runGoList("{{if .DepsErrors}}{{else}}error{{end}}", pkgpath, flags...) + out, err := runGoList("{{if .DepsErrors}}{{else}}error{{end}}", []string{pkgpath}, flags...) if err != nil { // if go list failed, assume dependencies are broken return false From 7d3c504c3c6c2a0dcecb0776c41b7fa0c7994a97 Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Thu, 2 Sep 2021 00:13:15 -0700 Subject: [PATCH 4/5] Fix godoc --- extractor/util/util.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extractor/util/util.go b/extractor/util/util.go index 24aebbad5e1..90b5b5a6813 100644 --- a/extractor/util/util.go +++ b/extractor/util/util.go @@ -58,9 +58,9 @@ type PkgInfo struct { ModDir string // the module directory containing this package, empty if not a module } -// GetPkgsInfo gets absolute directories of module and package root, where packages are denoted by `patterns`. -// It passes the `go list` the flags specified by `flags`. -// If `includingDeps`, all dependencies will also be included. see `go help list` for `-deps` flag +// GetPkgsInfo gets the absolute module and package root directories for the packages matched by the +// patterns `patterns`. It passes to `go list` the flags specified by `flags`. If `includingDeps` +// is true, all dependencies will also be included. func GetPkgsInfo(patterns []string, includingDeps bool, flags ...string) (map[string]*PkgInfo, error) { // enable module mode so that we can find a module root if it exists, even if go module support is // disabled by a build From f9ce06b4c0708abfff13ff7cb910fd0cf08dd603 Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Thu, 2 Sep 2021 00:13:54 -0700 Subject: [PATCH 5/5] Check for nil when getting package info --- extractor/extractor.go | 9 ++++++++- extractor/util/util.go | 10 ++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/extractor/extractor.go b/extractor/extractor.go index 50bf9d78c96..409ff8d7ae3 100644 --- a/extractor/extractor.go +++ b/extractor/extractor.go @@ -148,7 +148,14 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error { for _, pkg := range pkgs { info := pkgInfos[pkg.PkgPath] - if info.PkgDir == "" { + if info == nil { + var err error + info, err = util.GetPkgInfo(pkg.PkgPath) + if err != nil { + log.Fatalf("Unable to get a source directory for input package %s: %s", pkg.PkgPath, err) + } + } + if info == nil || info.PkgDir == "" { log.Fatalf("Unable to get a source directory for input package %s.", pkg.PkgPath) } wantedRoots[info.PkgDir] = true diff --git a/extractor/util/util.go b/extractor/util/util.go index 90b5b5a6813..41b3d5ca434 100644 --- a/extractor/util/util.go +++ b/extractor/util/util.go @@ -114,6 +114,16 @@ func GetPkgsInfo(patterns []string, includingDeps bool, flags ...string) (map[st return pkgInfoMapping, nil } +// GetPkgsInfo gets the absolute module and package root directories for the package `pkg`, passing +// the internal `go list` command the flags specified by `flags`. +func GetPkgInfo(pkg string, flags ...string) (*PkgInfo, error) { + pkgInfos, err := GetPkgsInfo([]string{pkg}, false, flags...) + if err != nil { + return nil, err + } + return pkgInfos[pkg], nil +} + // DepErrors checks there are any errors resolving dependencies for `pkgpath`. It passes the `go // list` command the flags specified by `flags`. func DepErrors(pkgpath string, flags ...string) bool {