From 0aa46becf95e27fcc4556d43fa7e58c092bb0cfc Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Wed, 5 Feb 2020 17:58:46 -0800 Subject: [PATCH 1/4] extractor: Use -mod=vendor when a vendor directory exists --- extractor/cli/go-autobuilder/go-autobuilder.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/extractor/cli/go-autobuilder/go-autobuilder.go b/extractor/cli/go-autobuilder/go-autobuilder.go index c860f9669c6..136136bae3b 100644 --- a/extractor/cli/go-autobuilder/go-autobuilder.go +++ b/extractor/cli/go-autobuilder/go-autobuilder.go @@ -167,6 +167,10 @@ func main() { log.Println("Found glide.yaml, enabling go modules") } + // if a vendor/modules.txt file exists, we assume that there are vendored Go dependencies, and + // skip the dependency installation step and run the extractor with `-mod=vendor` + hasVendor := fileExists("vendor/modules.txt") + // if `LGTM_INDEX_NEED_GOPATH` is set, it overrides the value for `needGopath` inferred above if needGopathOverride := os.Getenv("LGTM_INDEX_NEED_GOPATH"); needGopathOverride != "" { inLGTM = true @@ -274,7 +278,7 @@ func main() { // check whether an explicit dependency installation command was provided inst := os.Getenv("LGTM_INDEX_BUILD_COMMAND") var install *exec.Cmd - if inst == "" { + if inst == "" && !hasVendor { // if there is a build file, run the corresponding build tool buildSucceeded := tryBuild("Makefile", "make") || tryBuild("makefile", "make") || @@ -329,7 +333,7 @@ func main() { log.Println("Installing dependencies using `go get -v ./...`.") } } - } else { + } else if inst != "" { // write custom build commands into a script, then run it var ( ext = "" @@ -382,7 +386,13 @@ func main() { } log.Printf("Running extractor command '%s ./...' from directory '%s'.\n", extractor, cwd) - cmd := exec.Command(extractor, "./...") + var cmd *exec.Cmd + // check for `vendor/modules.txt` and not just `vendor` in order to distinguish non-go vendor dirs + if depMode == GoGetWithModules && hasVendor { + cmd = exec.Command(extractor, "-mod=vendor", "./...") + } else { + cmd = exec.Command(extractor, "./...") + } cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr err = cmd.Run() From 57b874e047858781268197197868bfd819d89ff4 Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Wed, 11 Mar 2020 02:59:33 -0700 Subject: [PATCH 2/4] extractor: Only skip dependency installation when vendor folder is detected --- .../cli/go-autobuilder/go-autobuilder.go | 76 ++++++++++--------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/extractor/cli/go-autobuilder/go-autobuilder.go b/extractor/cli/go-autobuilder/go-autobuilder.go index 136136bae3b..0bdaf5770c0 100644 --- a/extractor/cli/go-autobuilder/go-autobuilder.go +++ b/extractor/cli/go-autobuilder/go-autobuilder.go @@ -278,7 +278,7 @@ func main() { // check whether an explicit dependency installation command was provided inst := os.Getenv("LGTM_INDEX_BUILD_COMMAND") var install *exec.Cmd - if inst == "" && !hasVendor { + if inst == "" { // if there is a build file, run the corresponding build tool buildSucceeded := tryBuild("Makefile", "make") || tryBuild("makefile", "make") || @@ -288,52 +288,56 @@ func main() { tryBuild("build.sh", "./build.sh") if !buildSucceeded { - // automatically determine command to install dependencies - if depMode == Dep { - // set up the dep cache if SEMMLE_CACHE is set - cacheDir := os.Getenv("SEMMLE_CACHE") - if cacheDir != "" { - depCacheDir := filepath.Join(cacheDir, "go", "dep") - log.Printf("Attempting to create dep cache dir %s\n", depCacheDir) - err := os.MkdirAll(depCacheDir, 0755) - if err != nil { - log.Printf("Failed to create dep cache directory: %s\n", err.Error()) - } else { - log.Printf("Setting dep cache directory to %s\n", depCacheDir) - err = os.Setenv("DEPCACHEDIR", depCacheDir) + if hasVendor { + log.Printf("Skipping depedency installation because a Go vendor directory was found.") + } else { + // automatically determine command to install dependencies + if depMode == Dep { + // set up the dep cache if SEMMLE_CACHE is set + cacheDir := os.Getenv("SEMMLE_CACHE") + if cacheDir != "" { + depCacheDir := filepath.Join(cacheDir, "go", "dep") + log.Printf("Attempting to create dep cache dir %s\n", depCacheDir) + err := os.MkdirAll(depCacheDir, 0755) if err != nil { - log.Println("Failed to set dep cache directory") + log.Printf("Failed to create dep cache directory: %s\n", err.Error()) } else { - err = os.Setenv("DEPCACHEAGE", "720h") // 30 days + log.Printf("Setting dep cache directory to %s\n", depCacheDir) + err = os.Setenv("DEPCACHEDIR", depCacheDir) if err != nil { - log.Println("Failed to set dep cache age") + log.Println("Failed to set dep cache directory") + } else { + err = os.Setenv("DEPCACHEAGE", "720h") // 30 days + if err != nil { + log.Println("Failed to set dep cache age") + } } } } - } - if fileExists("Gopkg.lock") { - // if Gopkg.lock exists, don't update it and only vendor dependencies - install = exec.Command("dep", "ensure", "-v", "-vendor-only") + if fileExists("Gopkg.lock") { + // if Gopkg.lock exists, don't update it and only vendor dependencies + install = exec.Command("dep", "ensure", "-v", "-vendor-only") + } else { + install = exec.Command("dep", "ensure", "-v") + } + log.Println("Installing dependencies using `dep ensure`.") + } else if depMode == Glide { + install = exec.Command("glide", "install") + log.Println("Installing dependencies using `glide install`") } else { - install = exec.Command("dep", "ensure", "-v") - } - log.Println("Installing dependencies using `dep ensure`.") - } else if depMode == Glide { - install = exec.Command("glide", "install") - log.Println("Installing dependencies using `glide install`") - } else { - if depMode == GoGetWithModules { - // enable go modules if used - os.Setenv("GO111MODULE", "on") - } + if depMode == GoGetWithModules { + // enable go modules if used + os.Setenv("GO111MODULE", "on") + } - // get dependencies - install = exec.Command("go", "get", "-v", "./...") - log.Println("Installing dependencies using `go get -v ./...`.") + // get dependencies + install = exec.Command("go", "get", "-v", "./...") + log.Println("Installing dependencies using `go get -v ./...`.") + } } } - } else if inst != "" { + } else { // write custom build commands into a script, then run it var ( ext = "" From 1f83aa4586b8749fb01eabb560116dfd59a264ca Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Wed, 11 Mar 2020 03:10:35 -0700 Subject: [PATCH 3/4] Add a -mod=vendor change note --- change-notes/1.24/extractor-go.md | 1 + 1 file changed, 1 insertion(+) diff --git a/change-notes/1.24/extractor-go.md b/change-notes/1.24/extractor-go.md index 744d4732f86..02f65fc5093 100644 --- a/change-notes/1.24/extractor-go.md +++ b/change-notes/1.24/extractor-go.md @@ -6,3 +6,4 @@ * The autobuilder now runs Makefiles or custom build scripts present in the codebase to install dependencies. * The extractor now supports extracting go.mod files, enabling queries on dependencies and their versions. +* The extractor now attempts to automatically detect when a dependencies have been vendored and use `-mod=vendor` appropriately. From 5056b5f16146ea6bd8a2c78858a54919de1cc3a9 Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Wed, 11 Mar 2020 03:26:18 -0700 Subject: [PATCH 4/4] Apply review comments. Co-Authored-By: Max Schaefer <54907921+max-schaefer@users.noreply.github.com> --- change-notes/1.24/extractor-go.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/change-notes/1.24/extractor-go.md b/change-notes/1.24/extractor-go.md index 02f65fc5093..3699f33d1ba 100644 --- a/change-notes/1.24/extractor-go.md +++ b/change-notes/1.24/extractor-go.md @@ -6,4 +6,4 @@ * The autobuilder now runs Makefiles or custom build scripts present in the codebase to install dependencies. * The extractor now supports extracting go.mod files, enabling queries on dependencies and their versions. -* The extractor now attempts to automatically detect when a dependencies have been vendored and use `-mod=vendor` appropriately. +* The autobuilder now attempts to automatically detect when dependencies have been vendored and use `-mod=vendor` appropriately.