Merge pull request #19248 from github/mbg/go/private-registries

Go: Support private registries via `GOPROXY`

go/extractor/cli/go-autobuilder/go-autobuilder.go

@@ -67,13 +67,13 @@ func restoreRepoLayout(fromDir string, dirEntries []string, scratchDirName strin
 
 // addVersionToMod add a go version directive, e.g. `go 1.14` to a `go.mod` file.
 func addVersionToMod(version string) bool {
-	cmd := exec.Command("go", "mod", "edit", "-go="+version)
+	cmd := toolchain.GoCommand("mod", "edit", "-go="+version)
 	return util.RunCmd(cmd)
 }
 
 // checkVendor tests to see whether a vendor directory is inconsistent according to the go frontend
 func checkVendor() bool {
-	vendorCheckCmd := exec.Command("go", "list", "-mod=vendor", "./...")
+	vendorCheckCmd := toolchain.GoCommand("list", "-mod=vendor", "./...")
 	outp, err := vendorCheckCmd.CombinedOutput()
 	if err != nil {
 		badVendorRe := regexp.MustCompile(`(?m)^go: inconsistent vendoring in .*:$`)
@@ -438,7 +438,7 @@ func installDependencies(workspace project.GoWorkspace) {
 				util.RunCmd(vendor)
 			}
 
-			install = exec.Command("go", "get", "-v", "./...")
+			install = toolchain.GoCommand("get", "-v", "./...")
 			install.Dir = path
 			log.Printf("Installing dependencies using `go get -v ./...` in `%s`.\n", path)
 			util.RunCmd(install)

go/extractor/toolchain/toolchain.go

@@ -137,9 +137,16 @@ func SupportsWorkspaces() bool {
 	return GetEnvGoSemVer().IsAtLeast(V1_18)
 }
 
+// Constructs a `*exec.Cmd` for `go` with the specified arguments.
+func GoCommand(arg ...string) *exec.Cmd {
+	cmd := exec.Command("go", arg...)
+	util.ApplyProxyEnvVars(cmd)
+	return cmd
+}
+
 // Run `go mod tidy -e` in the directory given by `path`.
 func TidyModule(path string) *exec.Cmd {
-	cmd := exec.Command("go", "mod", "tidy", "-e")
+	cmd := GoCommand("mod", "tidy", "-e")
 	cmd.Dir = path
 	return cmd
 }
@@ -159,21 +166,21 @@ func InitModule(path string) *exec.Cmd {
 		}
 	}
 
-	modInit := exec.Command("go", "mod", "init", moduleName)
+	modInit := GoCommand("mod", "init", moduleName)
 	modInit.Dir = path
 	return modInit
 }
 
 // Constructs a command to run `go mod vendor -e` in the directory given by `path`.
 func VendorModule(path string) *exec.Cmd {
-	modVendor := exec.Command("go", "mod", "vendor", "-e")
+	modVendor := GoCommand("mod", "vendor", "-e")
 	modVendor.Dir = path
 	return modVendor
 }
 
 // Constructs a command to run `go version`.
 func Version() *exec.Cmd {
-	version := exec.Command("go", "version")
+	version := GoCommand("version")
 	return version
 }
 
@@ -209,7 +216,7 @@ func RunListWithEnv(format string, patterns []string, additionalEnv []string, fl
 func ListWithEnv(format string, patterns []string, additionalEnv []string, flags ...string) *exec.Cmd {
 	args := append([]string{"list", "-e", "-f", format}, flags...)
 	args = append(args, patterns...)
-	cmd := exec.Command("go", args...)
+	cmd := GoCommand(args...)
 	cmd.Env = append(os.Environ(), additionalEnv...)
 	return cmd
 }

go/extractor/util/BUILD.bazel

@@ -6,6 +6,7 @@ go_library(
     name = "util",
     srcs = [
         "extractvendordirs.go",
+        "registryproxy.go",
         "semver.go",
         "util.go",
     ],
@@ -17,6 +18,7 @@ go_library(
 go_test(
     name = "util_test",
     srcs = [
+        "registryproxy_test.go",
         "semver_test.go",
         "util_test.go",
     ],

go/extractor/util/registryproxy.go

@@ -0,0 +1,134 @@
+package util
+
+import (
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+const PROXY_HOST = "CODEQL_PROXY_HOST"
+const PROXY_PORT = "CODEQL_PROXY_PORT"
+const PROXY_CA_CERTIFICATE = "CODEQL_PROXY_CA_CERTIFICATE"
+const PROXY_URLS = "CODEQL_PROXY_URLS"
+const GOPROXY_SERVER = "goproxy_server"
+
+type RegistryConfig struct {
+	Type string `json:"type"`
+	URL  string `json:"url"`
+}
+
+// The address of the proxy including protocol and port (e.g. http://localhost:1234)
+var proxy_address string
+
+// The path to the temporary file that stores the proxy certificate, if any.
+var proxy_cert_file string
+
+// An array of registry configurations that are relevant to Go.
+// This excludes other registry configurations that may be available, but are not relevant to Go.
+var proxy_configs []RegistryConfig
+
+// Stores the environment variables that we wish to pass on to `go` commands.
+var proxy_vars []string = nil
+
+// Keeps track of whether we have inspected the proxy environment variables.
+// Needed since `proxy_vars` may be nil either way.
+var proxy_vars_checked bool = false
+
+// Tries to parse the given string value into an array of RegistryConfig values.
+func parseRegistryConfigs(str string) ([]RegistryConfig, error) {
+	var configs []RegistryConfig
+	if err := json.Unmarshal([]byte(str), &configs); err != nil {
+		return nil, err
+	}
+
+	return configs, nil
+}
+
+func getEnvVars() []string {
+	var result []string
+
+	if proxy_host, proxy_host_set := os.LookupEnv(PROXY_HOST); proxy_host_set {
+		if proxy_port, proxy_port_set := os.LookupEnv(PROXY_PORT); proxy_port_set {
+			proxy_address = fmt.Sprintf("http://%s:%s", proxy_host, proxy_port)
+			result = append(result, fmt.Sprintf("HTTP_PROXY=%s", proxy_address), fmt.Sprintf("HTTPS_PROXY=%s", proxy_address))
+
+			slog.Info("Found private registry proxy", slog.String("proxy_address", proxy_address))
+		}
+	}
+
+	if proxy_cert, proxy_cert_set := os.LookupEnv(PROXY_CA_CERTIFICATE); proxy_cert_set {
+		// Write the certificate to a temporary file
+		slog.Info("Found certificate")
+
+		f, err := os.CreateTemp("", "codeql-proxy.crt")
+		if err != nil {
+			slog.Error("Failed to create temporary file for the proxy certificate", slog.String("error", err.Error()))
+		}
+
+		_, err = f.WriteString(proxy_cert)
+		if err != nil {
+			slog.Error("Failed to write to the temporary certificate file", slog.String("error", err.Error()))
+		}
+
+		err = f.Close()
+		if err != nil {
+			slog.Error("Failed to close the temporary certificate file", slog.String("error", err.Error()))
+		} else {
+			proxy_cert_file = f.Name()
+			result = append(result, fmt.Sprintf("SSL_CERT_FILE=%s", proxy_cert_file))
+		}
+	}
+
+	if proxy_urls, proxy_urls_set := os.LookupEnv(PROXY_URLS); proxy_urls_set {
+		val, err := parseRegistryConfigs(proxy_urls)
+		if err != nil {
+			slog.Error("Unable to parse proxy configurations", slog.String("error", err.Error()))
+		} else {
+			// We only care about private registry configurations that are relevant to Go and
+			// filter others out at this point.
+			for _, cfg := range val {
+				if cfg.Type == GOPROXY_SERVER {
+					proxy_configs = append(proxy_configs, cfg)
+					slog.Info("Found GOPROXY server", slog.String("url", cfg.URL))
+				}
+			}
+
+			if len(proxy_configs) > 0 {
+				goproxy_val := "https://proxy.golang.org,direct"
+
+				for _, cfg := range proxy_configs {
+					goproxy_val = cfg.URL + "," + goproxy_val
+				}
+
+				result = append(result, fmt.Sprintf("GOPROXY=%s", goproxy_val), "GOPRIVATE=", "GONOPROXY=")
+			}
+		}
+	}
+
+	return result
+}
+
+// Applies private package proxy related environment variables to `cmd`.
+func ApplyProxyEnvVars(cmd *exec.Cmd) {
+	slog.Debug(
+		"Applying private registry proxy environment variables",
+		slog.String("cmd_args", strings.Join(cmd.Args, " ")),
+	)
+
+	// If we haven't done so yet, check whether the proxy environment variables are set
+	// and extract information from them.
+	if !proxy_vars_checked {
+		proxy_vars = getEnvVars()
+		proxy_vars_checked = true
+	}
+
+	// If the proxy is configured, `proxy_vars` will be not `nil`. We append those
+	// variables to the existing environment to preserve those environment variables.
+	// If `cmd.Env` is not changed, then the existing environment is also preserved.
+	if proxy_vars != nil {
+		cmd.Env = append(os.Environ(), proxy_vars...)
+	}
+}

go/extractor/util/registryproxy_test.go

@@ -0,0 +1,49 @@
+package util
+
+import (
+	"testing"
+)
+
+func parseRegistryConfigsFail(t *testing.T, str string) {
+	_, err := parseRegistryConfigs(str)
+
+	if err == nil {
+		t.Fatal("Expected `parseRegistryConfigs` to fail, but it succeeded.")
+	}
+}
+
+func parseRegistryConfigsSuccess(t *testing.T, str string) []RegistryConfig {
+	val, err := parseRegistryConfigs(str)
+
+	if err != nil {
+		t.Fatalf("Expected `parseRegistryConfigs` to succeed, but it failed: %s", err.Error())
+	}
+
+	return val
+}
+
+func TestParseRegistryConfigs(t *testing.T) {
+	parseRegistryConfigsFail(t, "")
+
+	empty := parseRegistryConfigsSuccess(t, "[]")
+
+	if len(empty) != 0 {
+		t.Fatal("Expected `parseRegistryConfigs(\"[]\")` to return no configurations, but got some.")
+	}
+
+	single := parseRegistryConfigsSuccess(t, "[{ \"type\": \"goproxy_server\", \"url\": \"https://proxy.example.com/mod\" }]")
+
+	if len(single) != 1 {
+		t.Fatalf("Expected `parseRegistryConfigs` to return one configuration, but got %d.", len(single))
+	}
+
+	first := single[0]
+
+	if first.Type != "goproxy_server" {
+		t.Fatalf("Expected `Type` to be `goproxy_server`, but got `%s`", first.Type)
+	}
+
+	if first.URL != "https://proxy.example.com/mod" {
+		t.Fatalf("Expected `URL` to be `https://proxy.example.com/mod`, but got `%s`", first.URL)
+	}
+}