Go: Move project analysis code to separate file

This commit is contained in:
Michael B. Gale
2024-01-23 13:44:11 +00:00
parent 0dc3c847bc
commit ee36e7424a
2 changed files with 259 additions and 245 deletions

View File

@@ -9,13 +9,13 @@ import (
"path/filepath"
"regexp"
"runtime"
"sort"
"strings"
"golang.org/x/mod/semver"
"github.com/github/codeql-go/extractor/autobuilder"
"github.com/github/codeql-go/extractor/diagnostics"
"github.com/github/codeql-go/extractor/project"
"github.com/github/codeql-go/extractor/toolchain"
"github.com/github/codeql-go/extractor/util"
)
@@ -142,59 +142,6 @@ func restoreRepoLayout(fromDir string, dirEntries []string, scratchDirName strin
}
}
// DependencyInstallerMode is an enum describing how dependencies should be installed
type DependencyInstallerMode int
const (
// GoGetNoModules represents dependency installation using `go get` without modules
GoGetNoModules DependencyInstallerMode = iota
// GoGetWithModules represents dependency installation using `go get` with modules
GoGetWithModules
// Dep represent dependency installation using `dep ensure`
Dep
// Glide represents dependency installation using `glide install`
Glide
)
// ModMode corresponds to the possible values of the -mod flag for the Go compiler
type ModMode int
const (
ModUnset ModMode = iota
ModReadonly
ModMod
ModVendor
)
// argsForGoVersion returns the arguments to pass to the Go compiler for the given `ModMode` and
// Go version
func (m ModMode) argsForGoVersion(version string) []string {
switch m {
case ModUnset:
return []string{}
case ModReadonly:
return []string{"-mod=readonly"}
case ModMod:
if !semver.IsValid(version) {
log.Fatalf("Invalid Go semver: '%s'", version)
}
if semver.Compare(version, "v1.14") < 0 {
return []string{} // -mod=mod is the default behaviour for go <= 1.13, and is not accepted as an argument
} else {
return []string{"-mod=mod"}
}
case ModVendor:
return []string{"-mod=vendor"}
}
return nil
}
type BuildInfo struct {
DepMode DependencyInstallerMode
ModMode ModMode
BaseDir string
}
// addVersionToMod add a go version directive, e.g. `go 1.14` to a `go.mod` file.
func addVersionToMod(version string) bool {
cmd := exec.Command("go", "mod", "edit", "-go="+version)
@@ -229,169 +176,9 @@ func getSourceDir() string {
return srcdir
}
func getDirs(paths []string) []string {
dirs := make([]string, len(paths))
for i, path := range paths {
dirs[i] = filepath.Dir(path)
}
return dirs
}
func checkDirsNested(inputDirs []string) (string, bool) {
// replace "." with "" so that we can check if all the paths are nested
dirs := make([]string, len(inputDirs))
for i, inputDir := range inputDirs {
if inputDir == "." {
dirs[i] = ""
} else {
dirs[i] = inputDir
}
}
// the paths were generated by a depth-first search so I think they might
// be sorted, but we sort them just in case
sort.Strings(dirs)
for _, dir := range dirs {
if !strings.HasPrefix(dir, dirs[0]) {
return "", false
}
}
return dirs[0], true
}
// Returns the directory to run the go build in and whether to use a go.mod
// file.
func findGoModFiles(emitDiagnostics bool) (baseDir string, useGoMod bool) {
goModPaths := util.FindAllFilesWithName(".", "go.mod", "vendor")
if len(goModPaths) == 0 {
baseDir = "."
useGoMod = false
return
}
goModDirs := getDirs(goModPaths)
if util.AnyGoFilesOutsideDirs(".", goModDirs...) {
if emitDiagnostics {
diagnostics.EmitGoFilesOutsideGoModules(goModPaths)
}
baseDir = "."
useGoMod = false
return
}
if len(goModPaths) > 1 {
// currently not supported
baseDir = "."
commonRoot, nested := checkDirsNested(goModDirs)
if nested && commonRoot == "" {
useGoMod = true
} else {
useGoMod = false
}
if emitDiagnostics {
if nested {
diagnostics.EmitMultipleGoModFoundNested(goModPaths)
} else {
diagnostics.EmitMultipleGoModFoundNotNested(goModPaths)
}
}
return
}
if emitDiagnostics {
if goModDirs[0] == "." {
diagnostics.EmitSingleRootGoModFound(goModPaths[0])
} else {
diagnostics.EmitSingleNonRootGoModFound(goModPaths[0])
}
}
baseDir = goModDirs[0]
useGoMod = true
return
}
// Returns the appropriate DependencyInstallerMode for the current project
func getDepMode(emitDiagnostics bool) (DependencyInstallerMode, string) {
bazelPaths := util.FindAllFilesWithName(".", "BUILD", "vendor")
bazelPaths = append(bazelPaths, util.FindAllFilesWithName(".", "BUILD.bazel", "vendor")...)
if len(bazelPaths) > 0 {
// currently not supported
if emitDiagnostics {
diagnostics.EmitBazelBuildFilesFound(bazelPaths)
}
}
goWorkPaths := util.FindAllFilesWithName(".", "go.work", "vendor")
if len(goWorkPaths) > 0 {
// currently not supported
if emitDiagnostics {
diagnostics.EmitGoWorkFound(goWorkPaths)
}
}
baseDir, useGoMod := findGoModFiles(emitDiagnostics)
if useGoMod {
log.Println("Found go.mod, enabling go modules")
return GoGetWithModules, baseDir
}
if util.FileExists("Gopkg.toml") {
if emitDiagnostics {
diagnostics.EmitGopkgTomlFound()
}
log.Println("Found Gopkg.toml, using dep instead of go get")
return Dep, "."
}
if util.FileExists("glide.yaml") {
if emitDiagnostics {
diagnostics.EmitGlideYamlFound()
}
log.Println("Found glide.yaml, using Glide instead of go get")
return Glide, "."
}
return GoGetNoModules, "."
}
type GoVersionInfo struct {
// The version string, if any
Version string
// A value indicating whether a version string was found
Found bool
}
// Tries to open `go.mod` and read a go directive, returning the version and whether it was found.
func tryReadGoDirective(buildInfo BuildInfo) GoVersionInfo {
if buildInfo.DepMode == GoGetWithModules {
versionRe := regexp.MustCompile(`(?m)^go[ \t\r]+([0-9]+\.[0-9]+(\.[0-9]+)?)$`)
goMod, err := os.ReadFile(filepath.Join(buildInfo.BaseDir, "go.mod"))
if err != nil {
log.Println("Failed to read go.mod to check for missing Go version")
} else {
matches := versionRe.FindSubmatch(goMod)
if matches != nil {
if len(matches) > 1 {
return GoVersionInfo{string(matches[1]), true}
}
}
}
}
return GoVersionInfo{"", false}
}
// Returns the appropriate ModMode for the current project
func getModMode(depMode DependencyInstallerMode, baseDir string) ModMode {
if depMode == GoGetWithModules {
// if a vendor/modules.txt file exists, we assume that there are vendored Go dependencies, and
// skip the dependency installation step and run the extractor with `-mod=vendor`
if util.FileExists(filepath.Join(baseDir, "vendor", "modules.txt")) {
return ModVendor
} else if util.DirExists(filepath.Join(baseDir, "vendor")) {
return ModMod
}
}
return ModUnset
}
// fixGoVendorIssues fixes issues with go vendor for go version >= 1.14
func fixGoVendorIssues(buildInfo *BuildInfo, goModVersionFound bool) {
if buildInfo.ModMode == ModVendor {
func fixGoVendorIssues(buildInfo *project.BuildInfo, goModVersionFound bool) {
if buildInfo.ModMode == project.ModVendor {
// fix go vendor issues with go versions >= 1.14 when no go version is specified in the go.mod
// if this is the case, and dependencies were vendored with an old go version (and therefore
// do not contain a '## explicit' annotation, the go command will fail and refuse to do any
@@ -399,7 +186,7 @@ func fixGoVendorIssues(buildInfo *BuildInfo, goModVersionFound bool) {
//
// we work around this by adding an explicit go version of 1.13, which is the last version
// where this is not an issue
if buildInfo.DepMode == GoGetWithModules {
if buildInfo.DepMode == project.GoGetWithModules {
if !goModVersionFound {
// if the go.mod does not contain a version line
modulesTxt, err := os.ReadFile("vendor/modules.txt")
@@ -410,7 +197,7 @@ func fixGoVendorIssues(buildInfo *BuildInfo, goModVersionFound bool) {
log.Println("Adding a version directive to the go.mod file as the modules.txt does not have explicit annotations")
if !addVersionToMod("1.13") {
log.Println("Failed to add a version to the go.mod file to fix explicitly required package bug; not using vendored dependencies")
buildInfo.ModMode = ModMod
buildInfo.ModMode = project.ModMod
}
}
}
@@ -419,9 +206,9 @@ func fixGoVendorIssues(buildInfo *BuildInfo, goModVersionFound bool) {
}
// Determines whether the project needs a GOPATH set up
func getNeedGopath(buildInfo BuildInfo, importpath string) bool {
func getNeedGopath(buildInfo project.BuildInfo, importpath string) bool {
needGopath := true
if buildInfo.DepMode == GoGetWithModules {
if buildInfo.DepMode == project.GoGetWithModules {
needGopath = false
}
// if `LGTM_INDEX_NEED_GOPATH` is set, it overrides the value for `needGopath` inferred above
@@ -442,9 +229,9 @@ func getNeedGopath(buildInfo BuildInfo, importpath string) bool {
}
// Try to update `go.mod` and `go.sum` if the go version is >= 1.16.
func tryUpdateGoModAndGoSum(buildInfo BuildInfo) {
func tryUpdateGoModAndGoSum(buildInfo project.BuildInfo) {
// Go 1.16 and later won't automatically attempt to update go.mod / go.sum during package loading, so try to update them here:
if buildInfo.ModMode != ModVendor && buildInfo.DepMode == GoGetWithModules && semver.Compare(getEnvGoSemVer(), "v1.16") >= 0 {
if buildInfo.ModMode != project.ModVendor && buildInfo.DepMode == project.GoGetWithModules && semver.Compare(getEnvGoSemVer(), "v1.16") >= 0 {
// stat go.mod and go.sum
goModPath := filepath.Join(buildInfo.BaseDir, "go.mod")
beforeGoModFileInfo, beforeGoModErr := os.Stat(goModPath)
@@ -608,7 +395,7 @@ func setGopath(root string) {
// Try to build the project without custom commands. If that fails, return a boolean indicating
// that we should install dependencies ourselves.
func buildWithoutCustomCommands(modMode ModMode) bool {
func buildWithoutCustomCommands(modMode project.ModMode) bool {
shouldInstallDependencies := false
// try to build the project
buildSucceeded := autobuilder.Autobuild()
@@ -619,7 +406,7 @@ func buildWithoutCustomCommands(modMode ModMode) bool {
log.Println("Build failed, continuing to install dependencies.")
shouldInstallDependencies = true
} else if util.DepErrors("./...", modMode.argsForGoVersion(getEnvGoSemVer())...) {
} else if util.DepErrors("./...", modMode.ArgsForGoVersion(getEnvGoSemVer())...) {
log.Println("Dependencies are still not resolving after the build, continuing to install dependencies.")
shouldInstallDependencies = true
@@ -662,10 +449,10 @@ func buildWithCustomCommands(inst string) {
}
// Install dependencies using the given dependency installer mode.
func installDependencies(buildInfo BuildInfo) {
func installDependencies(buildInfo project.BuildInfo) {
// automatically determine command to install dependencies
var install *exec.Cmd
if buildInfo.DepMode == Dep {
if buildInfo.DepMode == project.Dep {
// set up the dep cache if SEMMLE_CACHE is set
cacheDir := os.Getenv("SEMMLE_CACHE")
if cacheDir != "" {
@@ -695,14 +482,14 @@ func installDependencies(buildInfo BuildInfo) {
install = exec.Command("dep", "ensure", "-v")
}
log.Println("Installing dependencies using `dep ensure`.")
} else if buildInfo.DepMode == Glide {
} else if buildInfo.DepMode == project.Glide {
install = exec.Command("glide", "install")
log.Println("Installing dependencies using `glide install`")
} else {
// explicitly set go module support
if buildInfo.DepMode == GoGetWithModules {
if buildInfo.DepMode == project.GoGetWithModules {
os.Setenv("GO111MODULE", "on")
} else if buildInfo.DepMode == GoGetNoModules {
} else if buildInfo.DepMode == project.GoGetNoModules {
os.Setenv("GO111MODULE", "off")
}
@@ -715,15 +502,15 @@ func installDependencies(buildInfo BuildInfo) {
}
// Run the extractor.
func extract(buildInfo BuildInfo) {
func extract(buildInfo project.BuildInfo) {
extractor, err := util.GetExtractorPath()
if err != nil {
log.Fatalf("Could not determine path of extractor: %v.\n", err)
}
extractorArgs := []string{}
if buildInfo.DepMode == GoGetWithModules {
extractorArgs = append(extractorArgs, buildInfo.ModMode.argsForGoVersion(getEnvGoSemVer())...)
if buildInfo.DepMode == project.GoGetWithModules {
extractorArgs = append(extractorArgs, buildInfo.ModMode.ArgsForGoVersion(getEnvGoSemVer())...)
}
extractorArgs = append(extractorArgs, "./...")
@@ -738,12 +525,6 @@ func extract(buildInfo BuildInfo) {
}
}
func getBuildInfo(emitDiagnostics bool) BuildInfo {
depMode, baseDir := getDepMode(true)
modMode := getModMode(depMode, baseDir)
return BuildInfo{depMode, modMode, baseDir}
}
// Build the project and run the extractor.
func installDependenciesAndBuild() {
log.Printf("Autobuilder was built with %s, environment has %s\n", runtime.Version(), toolchain.GetEnvGoVersion())
@@ -755,12 +536,12 @@ func installDependenciesAndBuild() {
// determine how to install dependencies and whether a GOPATH needs to be set up before
// extraction
buildInfo := getBuildInfo(true)
buildInfo := project.GetBuildInfo(true)
if _, present := os.LookupEnv("GO111MODULE"); !present {
os.Setenv("GO111MODULE", "auto")
}
goVersionInfo := tryReadGoDirective(buildInfo)
goVersionInfo := project.TryReadGoDirective(buildInfo)
// This diagnostic is not required if the system Go version is 1.21 or greater, since the
// Go tooling should install required Go versions as needed.
@@ -802,18 +583,18 @@ func installDependenciesAndBuild() {
buildWithCustomCommands(inst)
}
if buildInfo.ModMode == ModVendor {
if buildInfo.ModMode == project.ModVendor {
// test if running `go` with -mod=vendor works, and if it doesn't, try to fallback to -mod=mod
// or not set if the go version < 1.14. Note we check this post-build in case the build brings
// the vendor directory up to date.
if !checkVendor() {
buildInfo.ModMode = ModMod
buildInfo.ModMode = project.ModMod
log.Println("The vendor directory is not consistent with the go.mod; not using vendored dependencies.")
}
}
if shouldInstallDependencies {
if buildInfo.ModMode == ModVendor {
if buildInfo.ModMode == project.ModVendor {
log.Printf("Skipping dependency installation because a Go vendor directory was found.")
} else {
installDependencies(buildInfo)
@@ -1079,8 +860,8 @@ func (v versionInfo) String() string {
// Get the version of Go to install and output it to stdout as json.
func identifyEnvironment() {
var v versionInfo
buildInfo := getBuildInfo(false)
goVersionInfo := tryReadGoDirective(buildInfo)
buildInfo := project.GetBuildInfo(false)
goVersionInfo := project.TryReadGoDirective(buildInfo)
v.goModVersion, v.goModVersionFound = goVersionInfo.Version, goVersionInfo.Found
v.goEnvVersionFound = toolchain.IsInstalled()

View File

@@ -0,0 +1,233 @@
package project
import (
"log"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"github.com/github/codeql-go/extractor/diagnostics"
"github.com/github/codeql-go/extractor/util"
"golang.org/x/mod/semver"
)
func getDirs(paths []string) []string {
dirs := make([]string, len(paths))
for i, path := range paths {
dirs[i] = filepath.Dir(path)
}
return dirs
}
func checkDirsNested(inputDirs []string) (string, bool) {
// replace "." with "" so that we can check if all the paths are nested
dirs := make([]string, len(inputDirs))
for i, inputDir := range inputDirs {
if inputDir == "." {
dirs[i] = ""
} else {
dirs[i] = inputDir
}
}
// the paths were generated by a depth-first search so I think they might
// be sorted, but we sort them just in case
sort.Strings(dirs)
for _, dir := range dirs {
if !strings.HasPrefix(dir, dirs[0]) {
return "", false
}
}
return dirs[0], true
}
// Returns the directory to run the go build in and whether to use a go.mod
// file.
func findGoModFiles(emitDiagnostics bool) (baseDir string, useGoMod bool) {
goModPaths := util.FindAllFilesWithName(".", "go.mod", "vendor")
if len(goModPaths) == 0 {
baseDir = "."
useGoMod = false
return
}
goModDirs := getDirs(goModPaths)
if util.AnyGoFilesOutsideDirs(".", goModDirs...) {
if emitDiagnostics {
diagnostics.EmitGoFilesOutsideGoModules(goModPaths)
}
baseDir = "."
useGoMod = false
return
}
if len(goModPaths) > 1 {
// currently not supported
baseDir = "."
commonRoot, nested := checkDirsNested(goModDirs)
if nested && commonRoot == "" {
useGoMod = true
} else {
useGoMod = false
}
if emitDiagnostics {
if nested {
diagnostics.EmitMultipleGoModFoundNested(goModPaths)
} else {
diagnostics.EmitMultipleGoModFoundNotNested(goModPaths)
}
}
return
}
if emitDiagnostics {
if goModDirs[0] == "." {
diagnostics.EmitSingleRootGoModFound(goModPaths[0])
} else {
diagnostics.EmitSingleNonRootGoModFound(goModPaths[0])
}
}
baseDir = goModDirs[0]
useGoMod = true
return
}
// DependencyInstallerMode is an enum describing how dependencies should be installed
type DependencyInstallerMode int
const (
// GoGetNoModules represents dependency installation using `go get` without modules
GoGetNoModules DependencyInstallerMode = iota
// GoGetWithModules represents dependency installation using `go get` with modules
GoGetWithModules
// Dep represent dependency installation using `dep ensure`
Dep
// Glide represents dependency installation using `glide install`
Glide
)
// Returns the appropriate DependencyInstallerMode for the current project
func getDepMode(emitDiagnostics bool) (DependencyInstallerMode, string) {
bazelPaths := util.FindAllFilesWithName(".", "BUILD", "vendor")
bazelPaths = append(bazelPaths, util.FindAllFilesWithName(".", "BUILD.bazel", "vendor")...)
if len(bazelPaths) > 0 {
// currently not supported
if emitDiagnostics {
diagnostics.EmitBazelBuildFilesFound(bazelPaths)
}
}
goWorkPaths := util.FindAllFilesWithName(".", "go.work", "vendor")
if len(goWorkPaths) > 0 {
// currently not supported
if emitDiagnostics {
diagnostics.EmitGoWorkFound(goWorkPaths)
}
}
baseDir, useGoMod := findGoModFiles(emitDiagnostics)
if useGoMod {
log.Println("Found go.mod, enabling go modules")
return GoGetWithModules, baseDir
}
if util.FileExists("Gopkg.toml") {
if emitDiagnostics {
diagnostics.EmitGopkgTomlFound()
}
log.Println("Found Gopkg.toml, using dep instead of go get")
return Dep, "."
}
if util.FileExists("glide.yaml") {
if emitDiagnostics {
diagnostics.EmitGlideYamlFound()
}
log.Println("Found glide.yaml, using Glide instead of go get")
return Glide, "."
}
return GoGetNoModules, "."
}
// ModMode corresponds to the possible values of the -mod flag for the Go compiler
type ModMode int
const (
ModUnset ModMode = iota
ModReadonly
ModMod
ModVendor
)
// argsForGoVersion returns the arguments to pass to the Go compiler for the given `ModMode` and
// Go version
func (m ModMode) ArgsForGoVersion(version string) []string {
switch m {
case ModUnset:
return []string{}
case ModReadonly:
return []string{"-mod=readonly"}
case ModMod:
if !semver.IsValid(version) {
log.Fatalf("Invalid Go semver: '%s'", version)
}
if semver.Compare(version, "v1.14") < 0 {
return []string{} // -mod=mod is the default behaviour for go <= 1.13, and is not accepted as an argument
} else {
return []string{"-mod=mod"}
}
case ModVendor:
return []string{"-mod=vendor"}
}
return nil
}
// Returns the appropriate ModMode for the current project
func getModMode(depMode DependencyInstallerMode, baseDir string) ModMode {
if depMode == GoGetWithModules {
// if a vendor/modules.txt file exists, we assume that there are vendored Go dependencies, and
// skip the dependency installation step and run the extractor with `-mod=vendor`
if util.FileExists(filepath.Join(baseDir, "vendor", "modules.txt")) {
return ModVendor
} else if util.DirExists(filepath.Join(baseDir, "vendor")) {
return ModMod
}
}
return ModUnset
}
type BuildInfo struct {
DepMode DependencyInstallerMode
ModMode ModMode
BaseDir string
}
func GetBuildInfo(emitDiagnostics bool) BuildInfo {
depMode, baseDir := getDepMode(true)
modMode := getModMode(depMode, baseDir)
return BuildInfo{depMode, modMode, baseDir}
}
type GoVersionInfo struct {
// The version string, if any
Version string
// A value indicating whether a version string was found
Found bool
}
// Tries to open `go.mod` and read a go directive, returning the version and whether it was found.
func TryReadGoDirective(buildInfo BuildInfo) GoVersionInfo {
if buildInfo.DepMode == GoGetWithModules {
versionRe := regexp.MustCompile(`(?m)^go[ \t\r]+([0-9]+\.[0-9]+(\.[0-9]+)?)$`)
goMod, err := os.ReadFile(filepath.Join(buildInfo.BaseDir, "go.mod"))
if err != nil {
log.Println("Failed to read go.mod to check for missing Go version")
} else {
matches := versionRe.FindSubmatch(goMod)
if matches != nil {
if len(matches) > 1 {
return GoVersionInfo{string(matches[1]), true}
}
}
}
}
return GoVersionInfo{"", false}
}