diff --git a/go/extractor/cli/go-autobuilder/go-autobuilder.go b/go/extractor/cli/go-autobuilder/go-autobuilder.go index 578dab01592..5523d0da756 100644 --- a/go/extractor/cli/go-autobuilder/go-autobuilder.go +++ b/go/extractor/cli/go-autobuilder/go-autobuilder.go @@ -9,13 +9,13 @@ import ( "path/filepath" "regexp" "runtime" - "sort" "strings" "golang.org/x/mod/semver" "github.com/github/codeql-go/extractor/autobuilder" "github.com/github/codeql-go/extractor/diagnostics" + "github.com/github/codeql-go/extractor/project" "github.com/github/codeql-go/extractor/toolchain" "github.com/github/codeql-go/extractor/util" ) @@ -142,59 +142,6 @@ func restoreRepoLayout(fromDir string, dirEntries []string, scratchDirName strin } } -// DependencyInstallerMode is an enum describing how dependencies should be installed -type DependencyInstallerMode int - -const ( - // GoGetNoModules represents dependency installation using `go get` without modules - GoGetNoModules DependencyInstallerMode = iota - // GoGetWithModules represents dependency installation using `go get` with modules - GoGetWithModules - // Dep represent dependency installation using `dep ensure` - Dep - // Glide represents dependency installation using `glide install` - Glide -) - -// ModMode corresponds to the possible values of the -mod flag for the Go compiler -type ModMode int - -const ( - ModUnset ModMode = iota - ModReadonly - ModMod - ModVendor -) - -// argsForGoVersion returns the arguments to pass to the Go compiler for the given `ModMode` and -// Go version -func (m ModMode) argsForGoVersion(version string) []string { - switch m { - case ModUnset: - return []string{} - case ModReadonly: - return []string{"-mod=readonly"} - case ModMod: - if !semver.IsValid(version) { - log.Fatalf("Invalid Go semver: '%s'", version) - } - if semver.Compare(version, "v1.14") < 0 { - return []string{} // -mod=mod is the default behaviour for go <= 1.13, and is not accepted as an argument - } else { - return []string{"-mod=mod"} - } - case ModVendor: - return []string{"-mod=vendor"} - } - return nil -} - -type BuildInfo struct { - DepMode DependencyInstallerMode - ModMode ModMode - BaseDir string -} - // addVersionToMod add a go version directive, e.g. `go 1.14` to a `go.mod` file. func addVersionToMod(version string) bool { cmd := exec.Command("go", "mod", "edit", "-go="+version) @@ -229,169 +176,9 @@ func getSourceDir() string { return srcdir } -func getDirs(paths []string) []string { - dirs := make([]string, len(paths)) - for i, path := range paths { - dirs[i] = filepath.Dir(path) - } - return dirs -} - -func checkDirsNested(inputDirs []string) (string, bool) { - // replace "." with "" so that we can check if all the paths are nested - dirs := make([]string, len(inputDirs)) - for i, inputDir := range inputDirs { - if inputDir == "." { - dirs[i] = "" - } else { - dirs[i] = inputDir - } - } - // the paths were generated by a depth-first search so I think they might - // be sorted, but we sort them just in case - sort.Strings(dirs) - for _, dir := range dirs { - if !strings.HasPrefix(dir, dirs[0]) { - return "", false - } - } - return dirs[0], true -} - -// Returns the directory to run the go build in and whether to use a go.mod -// file. -func findGoModFiles(emitDiagnostics bool) (baseDir string, useGoMod bool) { - goModPaths := util.FindAllFilesWithName(".", "go.mod", "vendor") - if len(goModPaths) == 0 { - baseDir = "." - useGoMod = false - return - } - goModDirs := getDirs(goModPaths) - if util.AnyGoFilesOutsideDirs(".", goModDirs...) { - if emitDiagnostics { - diagnostics.EmitGoFilesOutsideGoModules(goModPaths) - } - baseDir = "." - useGoMod = false - return - } - if len(goModPaths) > 1 { - // currently not supported - baseDir = "." - commonRoot, nested := checkDirsNested(goModDirs) - if nested && commonRoot == "" { - useGoMod = true - } else { - useGoMod = false - } - if emitDiagnostics { - if nested { - diagnostics.EmitMultipleGoModFoundNested(goModPaths) - } else { - diagnostics.EmitMultipleGoModFoundNotNested(goModPaths) - } - } - return - } - if emitDiagnostics { - if goModDirs[0] == "." { - diagnostics.EmitSingleRootGoModFound(goModPaths[0]) - } else { - diagnostics.EmitSingleNonRootGoModFound(goModPaths[0]) - } - } - baseDir = goModDirs[0] - useGoMod = true - return -} - -// Returns the appropriate DependencyInstallerMode for the current project -func getDepMode(emitDiagnostics bool) (DependencyInstallerMode, string) { - bazelPaths := util.FindAllFilesWithName(".", "BUILD", "vendor") - bazelPaths = append(bazelPaths, util.FindAllFilesWithName(".", "BUILD.bazel", "vendor")...) - if len(bazelPaths) > 0 { - // currently not supported - if emitDiagnostics { - diagnostics.EmitBazelBuildFilesFound(bazelPaths) - } - } - - goWorkPaths := util.FindAllFilesWithName(".", "go.work", "vendor") - if len(goWorkPaths) > 0 { - // currently not supported - if emitDiagnostics { - diagnostics.EmitGoWorkFound(goWorkPaths) - } - } - - baseDir, useGoMod := findGoModFiles(emitDiagnostics) - if useGoMod { - log.Println("Found go.mod, enabling go modules") - return GoGetWithModules, baseDir - } - - if util.FileExists("Gopkg.toml") { - if emitDiagnostics { - diagnostics.EmitGopkgTomlFound() - } - log.Println("Found Gopkg.toml, using dep instead of go get") - return Dep, "." - } - - if util.FileExists("glide.yaml") { - if emitDiagnostics { - diagnostics.EmitGlideYamlFound() - } - log.Println("Found glide.yaml, using Glide instead of go get") - return Glide, "." - } - return GoGetNoModules, "." -} - -type GoVersionInfo struct { - // The version string, if any - Version string - // A value indicating whether a version string was found - Found bool -} - -// Tries to open `go.mod` and read a go directive, returning the version and whether it was found. -func tryReadGoDirective(buildInfo BuildInfo) GoVersionInfo { - if buildInfo.DepMode == GoGetWithModules { - versionRe := regexp.MustCompile(`(?m)^go[ \t\r]+([0-9]+\.[0-9]+(\.[0-9]+)?)$`) - goMod, err := os.ReadFile(filepath.Join(buildInfo.BaseDir, "go.mod")) - if err != nil { - log.Println("Failed to read go.mod to check for missing Go version") - } else { - matches := versionRe.FindSubmatch(goMod) - if matches != nil { - if len(matches) > 1 { - return GoVersionInfo{string(matches[1]), true} - } - } - } - } - return GoVersionInfo{"", false} -} - -// Returns the appropriate ModMode for the current project -func getModMode(depMode DependencyInstallerMode, baseDir string) ModMode { - if depMode == GoGetWithModules { - // if a vendor/modules.txt file exists, we assume that there are vendored Go dependencies, and - // skip the dependency installation step and run the extractor with `-mod=vendor` - if util.FileExists(filepath.Join(baseDir, "vendor", "modules.txt")) { - return ModVendor - } else if util.DirExists(filepath.Join(baseDir, "vendor")) { - return ModMod - } - } - return ModUnset -} - // fixGoVendorIssues fixes issues with go vendor for go version >= 1.14 -func fixGoVendorIssues(buildInfo *BuildInfo, goModVersionFound bool) { - if buildInfo.ModMode == ModVendor { +func fixGoVendorIssues(buildInfo *project.BuildInfo, goModVersionFound bool) { + if buildInfo.ModMode == project.ModVendor { // fix go vendor issues with go versions >= 1.14 when no go version is specified in the go.mod // if this is the case, and dependencies were vendored with an old go version (and therefore // do not contain a '## explicit' annotation, the go command will fail and refuse to do any @@ -399,7 +186,7 @@ func fixGoVendorIssues(buildInfo *BuildInfo, goModVersionFound bool) { // // we work around this by adding an explicit go version of 1.13, which is the last version // where this is not an issue - if buildInfo.DepMode == GoGetWithModules { + if buildInfo.DepMode == project.GoGetWithModules { if !goModVersionFound { // if the go.mod does not contain a version line modulesTxt, err := os.ReadFile("vendor/modules.txt") @@ -410,7 +197,7 @@ func fixGoVendorIssues(buildInfo *BuildInfo, goModVersionFound bool) { log.Println("Adding a version directive to the go.mod file as the modules.txt does not have explicit annotations") if !addVersionToMod("1.13") { log.Println("Failed to add a version to the go.mod file to fix explicitly required package bug; not using vendored dependencies") - buildInfo.ModMode = ModMod + buildInfo.ModMode = project.ModMod } } } @@ -419,9 +206,9 @@ func fixGoVendorIssues(buildInfo *BuildInfo, goModVersionFound bool) { } // Determines whether the project needs a GOPATH set up -func getNeedGopath(buildInfo BuildInfo, importpath string) bool { +func getNeedGopath(buildInfo project.BuildInfo, importpath string) bool { needGopath := true - if buildInfo.DepMode == GoGetWithModules { + if buildInfo.DepMode == project.GoGetWithModules { needGopath = false } // if `LGTM_INDEX_NEED_GOPATH` is set, it overrides the value for `needGopath` inferred above @@ -442,9 +229,9 @@ func getNeedGopath(buildInfo BuildInfo, importpath string) bool { } // Try to update `go.mod` and `go.sum` if the go version is >= 1.16. -func tryUpdateGoModAndGoSum(buildInfo BuildInfo) { +func tryUpdateGoModAndGoSum(buildInfo project.BuildInfo) { // Go 1.16 and later won't automatically attempt to update go.mod / go.sum during package loading, so try to update them here: - if buildInfo.ModMode != ModVendor && buildInfo.DepMode == GoGetWithModules && semver.Compare(getEnvGoSemVer(), "v1.16") >= 0 { + if buildInfo.ModMode != project.ModVendor && buildInfo.DepMode == project.GoGetWithModules && semver.Compare(getEnvGoSemVer(), "v1.16") >= 0 { // stat go.mod and go.sum goModPath := filepath.Join(buildInfo.BaseDir, "go.mod") beforeGoModFileInfo, beforeGoModErr := os.Stat(goModPath) @@ -608,7 +395,7 @@ func setGopath(root string) { // Try to build the project without custom commands. If that fails, return a boolean indicating // that we should install dependencies ourselves. -func buildWithoutCustomCommands(modMode ModMode) bool { +func buildWithoutCustomCommands(modMode project.ModMode) bool { shouldInstallDependencies := false // try to build the project buildSucceeded := autobuilder.Autobuild() @@ -619,7 +406,7 @@ func buildWithoutCustomCommands(modMode ModMode) bool { log.Println("Build failed, continuing to install dependencies.") shouldInstallDependencies = true - } else if util.DepErrors("./...", modMode.argsForGoVersion(getEnvGoSemVer())...) { + } else if util.DepErrors("./...", modMode.ArgsForGoVersion(getEnvGoSemVer())...) { log.Println("Dependencies are still not resolving after the build, continuing to install dependencies.") shouldInstallDependencies = true @@ -662,10 +449,10 @@ func buildWithCustomCommands(inst string) { } // Install dependencies using the given dependency installer mode. -func installDependencies(buildInfo BuildInfo) { +func installDependencies(buildInfo project.BuildInfo) { // automatically determine command to install dependencies var install *exec.Cmd - if buildInfo.DepMode == Dep { + if buildInfo.DepMode == project.Dep { // set up the dep cache if SEMMLE_CACHE is set cacheDir := os.Getenv("SEMMLE_CACHE") if cacheDir != "" { @@ -695,14 +482,14 @@ func installDependencies(buildInfo BuildInfo) { install = exec.Command("dep", "ensure", "-v") } log.Println("Installing dependencies using `dep ensure`.") - } else if buildInfo.DepMode == Glide { + } else if buildInfo.DepMode == project.Glide { install = exec.Command("glide", "install") log.Println("Installing dependencies using `glide install`") } else { // explicitly set go module support - if buildInfo.DepMode == GoGetWithModules { + if buildInfo.DepMode == project.GoGetWithModules { os.Setenv("GO111MODULE", "on") - } else if buildInfo.DepMode == GoGetNoModules { + } else if buildInfo.DepMode == project.GoGetNoModules { os.Setenv("GO111MODULE", "off") } @@ -715,15 +502,15 @@ func installDependencies(buildInfo BuildInfo) { } // Run the extractor. -func extract(buildInfo BuildInfo) { +func extract(buildInfo project.BuildInfo) { extractor, err := util.GetExtractorPath() if err != nil { log.Fatalf("Could not determine path of extractor: %v.\n", err) } extractorArgs := []string{} - if buildInfo.DepMode == GoGetWithModules { - extractorArgs = append(extractorArgs, buildInfo.ModMode.argsForGoVersion(getEnvGoSemVer())...) + if buildInfo.DepMode == project.GoGetWithModules { + extractorArgs = append(extractorArgs, buildInfo.ModMode.ArgsForGoVersion(getEnvGoSemVer())...) } extractorArgs = append(extractorArgs, "./...") @@ -738,12 +525,6 @@ func extract(buildInfo BuildInfo) { } } -func getBuildInfo(emitDiagnostics bool) BuildInfo { - depMode, baseDir := getDepMode(true) - modMode := getModMode(depMode, baseDir) - return BuildInfo{depMode, modMode, baseDir} -} - // Build the project and run the extractor. func installDependenciesAndBuild() { log.Printf("Autobuilder was built with %s, environment has %s\n", runtime.Version(), toolchain.GetEnvGoVersion()) @@ -755,12 +536,12 @@ func installDependenciesAndBuild() { // determine how to install dependencies and whether a GOPATH needs to be set up before // extraction - buildInfo := getBuildInfo(true) + buildInfo := project.GetBuildInfo(true) if _, present := os.LookupEnv("GO111MODULE"); !present { os.Setenv("GO111MODULE", "auto") } - goVersionInfo := tryReadGoDirective(buildInfo) + goVersionInfo := project.TryReadGoDirective(buildInfo) // This diagnostic is not required if the system Go version is 1.21 or greater, since the // Go tooling should install required Go versions as needed. @@ -802,18 +583,18 @@ func installDependenciesAndBuild() { buildWithCustomCommands(inst) } - if buildInfo.ModMode == ModVendor { + if buildInfo.ModMode == project.ModVendor { // test if running `go` with -mod=vendor works, and if it doesn't, try to fallback to -mod=mod // or not set if the go version < 1.14. Note we check this post-build in case the build brings // the vendor directory up to date. if !checkVendor() { - buildInfo.ModMode = ModMod + buildInfo.ModMode = project.ModMod log.Println("The vendor directory is not consistent with the go.mod; not using vendored dependencies.") } } if shouldInstallDependencies { - if buildInfo.ModMode == ModVendor { + if buildInfo.ModMode == project.ModVendor { log.Printf("Skipping dependency installation because a Go vendor directory was found.") } else { installDependencies(buildInfo) @@ -1079,8 +860,8 @@ func (v versionInfo) String() string { // Get the version of Go to install and output it to stdout as json. func identifyEnvironment() { var v versionInfo - buildInfo := getBuildInfo(false) - goVersionInfo := tryReadGoDirective(buildInfo) + buildInfo := project.GetBuildInfo(false) + goVersionInfo := project.TryReadGoDirective(buildInfo) v.goModVersion, v.goModVersionFound = goVersionInfo.Version, goVersionInfo.Found v.goEnvVersionFound = toolchain.IsInstalled() diff --git a/go/extractor/project/project.go b/go/extractor/project/project.go new file mode 100644 index 00000000000..baef8bdefb8 --- /dev/null +++ b/go/extractor/project/project.go @@ -0,0 +1,233 @@ +package project + +import ( + "log" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + + "github.com/github/codeql-go/extractor/diagnostics" + "github.com/github/codeql-go/extractor/util" + "golang.org/x/mod/semver" +) + +func getDirs(paths []string) []string { + dirs := make([]string, len(paths)) + for i, path := range paths { + dirs[i] = filepath.Dir(path) + } + return dirs +} + +func checkDirsNested(inputDirs []string) (string, bool) { + // replace "." with "" so that we can check if all the paths are nested + dirs := make([]string, len(inputDirs)) + for i, inputDir := range inputDirs { + if inputDir == "." { + dirs[i] = "" + } else { + dirs[i] = inputDir + } + } + // the paths were generated by a depth-first search so I think they might + // be sorted, but we sort them just in case + sort.Strings(dirs) + for _, dir := range dirs { + if !strings.HasPrefix(dir, dirs[0]) { + return "", false + } + } + return dirs[0], true +} + +// Returns the directory to run the go build in and whether to use a go.mod +// file. +func findGoModFiles(emitDiagnostics bool) (baseDir string, useGoMod bool) { + goModPaths := util.FindAllFilesWithName(".", "go.mod", "vendor") + if len(goModPaths) == 0 { + baseDir = "." + useGoMod = false + return + } + goModDirs := getDirs(goModPaths) + if util.AnyGoFilesOutsideDirs(".", goModDirs...) { + if emitDiagnostics { + diagnostics.EmitGoFilesOutsideGoModules(goModPaths) + } + baseDir = "." + useGoMod = false + return + } + if len(goModPaths) > 1 { + // currently not supported + baseDir = "." + commonRoot, nested := checkDirsNested(goModDirs) + if nested && commonRoot == "" { + useGoMod = true + } else { + useGoMod = false + } + if emitDiagnostics { + if nested { + diagnostics.EmitMultipleGoModFoundNested(goModPaths) + } else { + diagnostics.EmitMultipleGoModFoundNotNested(goModPaths) + } + } + return + } + if emitDiagnostics { + if goModDirs[0] == "." { + diagnostics.EmitSingleRootGoModFound(goModPaths[0]) + } else { + diagnostics.EmitSingleNonRootGoModFound(goModPaths[0]) + } + } + baseDir = goModDirs[0] + useGoMod = true + return +} + +// DependencyInstallerMode is an enum describing how dependencies should be installed +type DependencyInstallerMode int + +const ( + // GoGetNoModules represents dependency installation using `go get` without modules + GoGetNoModules DependencyInstallerMode = iota + // GoGetWithModules represents dependency installation using `go get` with modules + GoGetWithModules + // Dep represent dependency installation using `dep ensure` + Dep + // Glide represents dependency installation using `glide install` + Glide +) + +// Returns the appropriate DependencyInstallerMode for the current project +func getDepMode(emitDiagnostics bool) (DependencyInstallerMode, string) { + bazelPaths := util.FindAllFilesWithName(".", "BUILD", "vendor") + bazelPaths = append(bazelPaths, util.FindAllFilesWithName(".", "BUILD.bazel", "vendor")...) + if len(bazelPaths) > 0 { + // currently not supported + if emitDiagnostics { + diagnostics.EmitBazelBuildFilesFound(bazelPaths) + } + } + + goWorkPaths := util.FindAllFilesWithName(".", "go.work", "vendor") + if len(goWorkPaths) > 0 { + // currently not supported + if emitDiagnostics { + diagnostics.EmitGoWorkFound(goWorkPaths) + } + } + + baseDir, useGoMod := findGoModFiles(emitDiagnostics) + if useGoMod { + log.Println("Found go.mod, enabling go modules") + return GoGetWithModules, baseDir + } + + if util.FileExists("Gopkg.toml") { + if emitDiagnostics { + diagnostics.EmitGopkgTomlFound() + } + log.Println("Found Gopkg.toml, using dep instead of go get") + return Dep, "." + } + + if util.FileExists("glide.yaml") { + if emitDiagnostics { + diagnostics.EmitGlideYamlFound() + } + log.Println("Found glide.yaml, using Glide instead of go get") + return Glide, "." + } + return GoGetNoModules, "." +} + +// ModMode corresponds to the possible values of the -mod flag for the Go compiler +type ModMode int + +const ( + ModUnset ModMode = iota + ModReadonly + ModMod + ModVendor +) + +// argsForGoVersion returns the arguments to pass to the Go compiler for the given `ModMode` and +// Go version +func (m ModMode) ArgsForGoVersion(version string) []string { + switch m { + case ModUnset: + return []string{} + case ModReadonly: + return []string{"-mod=readonly"} + case ModMod: + if !semver.IsValid(version) { + log.Fatalf("Invalid Go semver: '%s'", version) + } + if semver.Compare(version, "v1.14") < 0 { + return []string{} // -mod=mod is the default behaviour for go <= 1.13, and is not accepted as an argument + } else { + return []string{"-mod=mod"} + } + case ModVendor: + return []string{"-mod=vendor"} + } + return nil +} + +// Returns the appropriate ModMode for the current project +func getModMode(depMode DependencyInstallerMode, baseDir string) ModMode { + if depMode == GoGetWithModules { + // if a vendor/modules.txt file exists, we assume that there are vendored Go dependencies, and + // skip the dependency installation step and run the extractor with `-mod=vendor` + if util.FileExists(filepath.Join(baseDir, "vendor", "modules.txt")) { + return ModVendor + } else if util.DirExists(filepath.Join(baseDir, "vendor")) { + return ModMod + } + } + return ModUnset +} + +type BuildInfo struct { + DepMode DependencyInstallerMode + ModMode ModMode + BaseDir string +} + +func GetBuildInfo(emitDiagnostics bool) BuildInfo { + depMode, baseDir := getDepMode(true) + modMode := getModMode(depMode, baseDir) + return BuildInfo{depMode, modMode, baseDir} +} + +type GoVersionInfo struct { + // The version string, if any + Version string + // A value indicating whether a version string was found + Found bool +} + +// Tries to open `go.mod` and read a go directive, returning the version and whether it was found. +func TryReadGoDirective(buildInfo BuildInfo) GoVersionInfo { + if buildInfo.DepMode == GoGetWithModules { + versionRe := regexp.MustCompile(`(?m)^go[ \t\r]+([0-9]+\.[0-9]+(\.[0-9]+)?)$`) + goMod, err := os.ReadFile(filepath.Join(buildInfo.BaseDir, "go.mod")) + if err != nil { + log.Println("Failed to read go.mod to check for missing Go version") + } else { + matches := versionRe.FindSubmatch(goMod) + if matches != nil { + if len(matches) > 1 { + return GoVersionInfo{string(matches[1]), true} + } + } + } + } + return GoVersionInfo{"", false} +}