Merge pull request #17227 from smowton/smowton/fix/baseline-vs-nonroot-vendor-dirs

Go / configure-baseline: account for multiple vendor directories and the `CODEQL_EXTRACTOR_GO_EXTRACT_VENDOR_DIRS` setting

go/BUILD.bazel

@@ -47,6 +47,7 @@ codeql_pkg_files(
         "//go/extractor/cli/go-autobuilder",
         "//go/extractor/cli/go-bootstrap",
         "//go/extractor/cli/go-build-runner",
+        "//go/extractor/cli/go-configure-baseline",
         "//go/extractor/cli/go-extractor",
         "//go/extractor/cli/go-gen-dbscheme",
         "//go/extractor/cli/go-tokenizer",

go/codeql-tools/baseline-config-empty.json

@@ -1,3 +0,0 @@
-{
-    "paths-ignore": []
-}

go/codeql-tools/baseline-config-vendor.json

@@ -1,5 +0,0 @@
-{
-    "paths-ignore": [
-        "vendor/**"
-    ]
-}

go/codeql-tools/configure-baseline.cmd

@@ -1,6 +1,4 @@
 @echo off
-if exist vendor\modules.txt (
-  type "%CODEQL_EXTRACTOR_GO_ROOT%\tools\baseline-config-vendor.json"
-) else (
-  type "%CODEQL_EXTRACTOR_GO_ROOT%\tools\baseline-config-empty.json"
-)
+
+type NUL && "%CODEQL_EXTRACTOR_GO_ROOT%/tools/%CODEQL_PLATFORM%/go-configure-baseline.exe"
+exit /b %ERRORLEVEL%

go/codeql-tools/configure-baseline.sh

@@ -1,7 +1,3 @@
 #!/bin/sh
 
-if [ -f vendor/modules.txt ]; then
-  cat "$CODEQL_EXTRACTOR_GO_ROOT/tools/baseline-config-vendor.json"
-else
-  cat "$CODEQL_EXTRACTOR_GO_ROOT/tools/baseline-config-empty.json"
-fi
+"$CODEQL_EXTRACTOR_GO_ROOT/tools/$CODEQL_PLATFORM/go-configure-baseline"

go/extractor/cli/go-configure-baseline/BUILD.bazel

@@ -0,0 +1,18 @@
+# generated running `bazel run //go/gazelle`, do not edit
+
+load("@rules_go//go:def.bzl", "go_library")
+load("//go:rules.bzl", "codeql_go_binary")
+
+go_library(
+    name = "go-configure-baseline_lib",
+    srcs = ["go-configure-baseline.go"],
+    importpath = "github.com/github/codeql-go/extractor/cli/go-configure-baseline",
+    visibility = ["//visibility:private"],
+    deps = ["//go/extractor/configurebaseline"],
+)
+
+codeql_go_binary(
+    name = "go-configure-baseline",
+    embed = [":go-configure-baseline_lib"],
+    visibility = ["//visibility:public"],
+)

go/extractor/cli/go-configure-baseline/go-configure-baseline.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/github/codeql-go/extractor/configurebaseline"
+)
+
+func main() {
+	jsonResult, err := configurebaseline.GetConfigBaselineAsJSON(".")
+	if err != nil {
+		panic(err)
+	} else {
+		fmt.Println(string(jsonResult))
+	}
+}

go/extractor/configurebaseline/BUILD.bazel

@@ -0,0 +1,11 @@
+# generated running `bazel run //go/gazelle`, do not edit
+
+load("@rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "configurebaseline",
+    srcs = ["configurebaseline.go"],
+    importpath = "github.com/github/codeql-go/extractor/configurebaseline",
+    visibility = ["//visibility:public"],
+    deps = ["//go/extractor/util"],
+)

go/extractor/configurebaseline/configurebaseline.go

@@ -0,0 +1,52 @@
+package configurebaseline
+
+import (
+	"encoding/json"
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+
+	"github.com/github/codeql-go/extractor/util"
+)
+
+func fileExists(path string) bool {
+	stat, err := os.Stat(path)
+	return err == nil && stat.Mode().IsRegular()
+}
+
+// Decides if `dirPath` is a vendor directory by testing whether it is called `vendor`
+// and contains a `modules.txt` file.
+func isGolangVendorDirectory(dirPath string) bool {
+	return filepath.Base(dirPath) == "vendor" && fileExists(filepath.Join(dirPath, "modules.txt"))
+}
+
+type BaselineConfig struct {
+	PathsIgnore []string `json:"paths-ignore"`
+}
+
+func GetConfigBaselineAsJSON(rootDir string) ([]byte, error) {
+	vendorDirs := make([]string, 0)
+
+	if util.IsVendorDirExtractionEnabled() {
+		// The user wants vendor directories scanned; emit an empty report.
+	} else {
+		filepath.WalkDir(rootDir, func(dirPath string, d fs.DirEntry, err error) error {
+			if err != nil {
+				// Ignore any unreadable paths -- if this script can't see it, very likely
+				// it will not be extracted either.
+				return nil
+			}
+			if isGolangVendorDirectory(dirPath) {
+				// Note that CodeQL expects a forward-slash-separated path, even on Windows.
+				vendorDirs = append(vendorDirs, path.Join(filepath.ToSlash(dirPath), "**"))
+				return filepath.SkipDir
+			} else {
+				return nil
+			}
+		})
+	}
+
+	outputStruct := BaselineConfig{PathsIgnore: vendorDirs}
+	return json.Marshal(outputStruct)
+}

go/extractor/extractor.go

@@ -199,7 +199,7 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 
 	// If CODEQL_EXTRACTOR_GO_EXTRACT_VENDOR_DIRS is "true", we extract `vendor` directories;
 	// otherwise (the default) is to exclude them from extraction
-	includeVendor := os.Getenv("CODEQL_EXTRACTOR_GO_EXTRACT_VENDOR_DIRS") == "true"
+	includeVendor := util.IsVendorDirExtractionEnabled()
 	if !includeVendor {
 		excludedDirs = append(excludedDirs, "vendor")
 	}

go/extractor/util/BUILD.bazel

@@ -5,6 +5,7 @@ load("@rules_go//go:def.bzl", "go_library", "go_test")
 go_library(
     name = "util",
     srcs = [
+        "extractvendordirs.go",
         "semver.go",
         "util.go",
     ],

go/extractor/util/extractvendordirs.go

@@ -0,0 +1,9 @@
+package util
+
+import (
+	"os"
+)
+
+func IsVendorDirExtractionEnabled() bool {
+	return os.Getenv("CODEQL_EXTRACTOR_GO_EXTRACT_VENDOR_DIRS") == "true"
+}

go/ql/integration-tests/all-platforms/go/configure-baseline/src/a/vendor/avendor.go

@@ -0,0 +1 @@
+package abc

go/ql/integration-tests/all-platforms/go/configure-baseline/src/b/vendor/bvendor.go

@@ -0,0 +1 @@
+package abc

go/ql/integration-tests/all-platforms/go/configure-baseline/src/c/vendor/cvendor.go

@@ -0,0 +1 @@
+package abc

go/ql/integration-tests/all-platforms/go/configure-baseline/src/root.go

@@ -0,0 +1 @@
+package abc

go/ql/integration-tests/all-platforms/go/configure-baseline/test.py

@@ -0,0 +1,9 @@
+import os.path
+import json
+
+def test(codeql, go):
+    codeql.database.init(source_root="src")
+    baseline_info_path = os.path.join("test-db", "baseline-info.json")
+    with open(baseline_info_path, "r") as f:
+        baseline_info = json.load(f)
+    assert set(baseline_info["languages"]["go"]["files"]) == set(["root.go", "c/vendor/cvendor.go"]), "Expected root.go and cvendor.go in baseline"

go/ql/lib/change-notes/2024-08-20-vendor-dirs-baseline.md

@@ -0,0 +1,4 @@
+---
+category: fix
+---
+* Golang vendor directories not at the root of a repository are now correctly excluded from the baseline Go file count. This means code coverage information will be more accurate.