diff --git a/change-notes/1.24/extractor-go.md b/change-notes/1.24/extractor-go.md index b7ae7ac1e25..7dae6360f8f 100644 --- a/change-notes/1.24/extractor-go.md +++ b/change-notes/1.24/extractor-go.md @@ -4,6 +4,9 @@ ## Changes to code extraction +* In resource-constrained environments, the environment variable `CODEQL_EXTRACTOR_GO_MAX_GOROUTINES` can be used to limit the + number of parallel goroutines started by the extractor, which reduces CPU and memory requirements. The default value for this + variable is 32. * The autobuilder now runs Makefiles or custom build scripts present in the codebase to install dependencies. The build command to invoke can be configured via `lgtm.yml`, or by setting the environment variable `CODEQL_EXTRACTOR_GO_BUILD_COMMAND`. * The autobuilder now attempts to automatically detect when dependencies have been vendored and use `-mod=vendor` appropriately. diff --git a/extractor/cli/go-autobuilder/go-autobuilder.go b/extractor/cli/go-autobuilder/go-autobuilder.go index 52f8df4b939..a64ad80bb56 100644 --- a/extractor/cli/go-autobuilder/go-autobuilder.go +++ b/extractor/cli/go-autobuilder/go-autobuilder.go @@ -11,6 +11,8 @@ import ( "regexp" "runtime" "strings" + + "github.com/github/codeql-go/extractor/util" ) func usage() { @@ -32,6 +34,11 @@ to 'false' disables the GOPATH set-up, CODEQL_EXTRACTOR_GO_BUILD_COMMAND (or alt LGTM_INDEX_BUILD_COMMAND), can be set to a newline-separated list of commands to run in order to install dependencies, and LGTM_INDEX_IMPORT_PATH can be used to override the package import path, which is otherwise inferred from the SEMMLE_REPO_URL environment variable. + +In resource-constrained environments, the environment variable CODEQL_EXTRACTOR_GO_MAX_GOROUTINES +(or its legacy alias SEMMLE_MAX_GOROUTINES) can be used to limit the number of parallel goroutines +started by the extractor, which reduces CPU and memory requirements. The default value for this +variable is 32. `, os.Args[0]) fmt.Fprintf(os.Stderr, "Usage:\n\n %s\n", os.Args[0]) @@ -276,10 +283,7 @@ func main() { } // check whether an explicit dependency installation command was provided - inst := os.Getenv("CODEQL_EXTRACTOR_GO_BUILD_COMMAND") - if inst == "" { - inst = os.Getenv("LGTM_INDEX_BUILD_COMMAND") - } + inst := util.Getenv("CODEQL_EXTRACTOR_GO_BUILD_COMMAND", "LGTM_INDEX_BUILD_COMMAND") var install *exec.Cmd if inst == "" { // if there is a build file, run the corresponding build tool diff --git a/extractor/extractor.go b/extractor/extractor.go index 5bec4cf943d..f59ee9c228c 100644 --- a/extractor/extractor.go +++ b/extractor/extractor.go @@ -20,6 +20,7 @@ import ( "github.com/github/codeql-go/extractor/dbscheme" "github.com/github/codeql-go/extractor/srcarchive" "github.com/github/codeql-go/extractor/trap" + "github.com/github/codeql-go/extractor/util" "golang.org/x/tools/go/packages" ) @@ -89,7 +90,10 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error { // available since Go 1.5, but is subject to change var maxgoroutines int - if maxgoroutines, err = strconv.Atoi(os.Getenv("SEMMLE_MAX_GOROUTINES")); err != nil { + if maxgoroutines, err = strconv.Atoi(util.Getenv( + "CODEQL_EXTRACTOR_GO_MAX_GOROUTINES", + "SEMMLE_MAX_GOROUTINES", + )); err != nil { maxgoroutines = 32 } else { log.Printf("Max goroutines set to %d", maxgoroutines) diff --git a/extractor/util/util.go b/extractor/util/util.go new file mode 100644 index 00000000000..fdbc76c35b6 --- /dev/null +++ b/extractor/util/util.go @@ -0,0 +1,20 @@ +package util + +import "os" + +// Getenv retrieves the value of the environment variable named by the key. +// If that variable is not present, it iterates over the given aliases until +// it finds one that is. If none are present, the empty string is returned. +func Getenv(key string, aliases ...string) string { + val := os.Getenv(key) + if val != "" { + return val + } + for _, alias := range aliases { + val = os.Getenv(alias) + if val != "" { + return val + } + } + return "" +}