From aff874e8352b27eaf6d569584fff04ac74429b74 Mon Sep 17 00:00:00 2001 From: Nick Rolfe Date: Wed, 3 Sep 2025 16:36:28 +0100 Subject: [PATCH] Go: merge with incoming path transformer when setting GOPATH --- go/extractor/cli/go-autobuilder/BUILD.bazel | 1 + .../cli/go-autobuilder/go-autobuilder.go | 23 ++++++-- go/extractor/srcarchive/projectlayout.go | 35 +++++++++--- go/extractor/srcarchive/projectlayout_test.go | 56 ++++++++++++++++--- go/extractor/srcarchive/srcarchive.go | 17 ++---- 5 files changed, 100 insertions(+), 32 deletions(-) diff --git a/go/extractor/cli/go-autobuilder/BUILD.bazel b/go/extractor/cli/go-autobuilder/BUILD.bazel index 0c705d5b194..003836b0874 100644 --- a/go/extractor/cli/go-autobuilder/BUILD.bazel +++ b/go/extractor/cli/go-autobuilder/BUILD.bazel @@ -12,6 +12,7 @@ go_library( "//go/extractor/autobuilder", "//go/extractor/diagnostics", "//go/extractor/project", + "//go/extractor/srcarchive", "//go/extractor/toolchain", "//go/extractor/util", ], diff --git a/go/extractor/cli/go-autobuilder/go-autobuilder.go b/go/extractor/cli/go-autobuilder/go-autobuilder.go index d9e294524e6..c22b356c4be 100644 --- a/go/extractor/cli/go-autobuilder/go-autobuilder.go +++ b/go/extractor/cli/go-autobuilder/go-autobuilder.go @@ -13,6 +13,7 @@ import ( "github.com/github/codeql-go/extractor/autobuilder" "github.com/github/codeql-go/extractor/diagnostics" "github.com/github/codeql-go/extractor/project" + "github.com/github/codeql-go/extractor/srcarchive" "github.com/github/codeql-go/extractor/toolchain" "github.com/github/codeql-go/extractor/util" ) @@ -283,7 +284,7 @@ func createPathTransformerFile(newdir string) *os.File { } // Writes the path transformer file -func writePathTransformerFile(pt *os.File, realSrc, root, newdir string) { +func writePathTransformerFile(pt *os.File, realSrc, newdir string) { _, err := pt.WriteString("#" + realSrc + "\n" + newdir + "//\n") if err != nil { log.Fatalf("Unable to write path transformer file: %s.", err.Error()) @@ -501,10 +502,6 @@ func installDependenciesAndBuild() { srcdir := getSourceDir() - // we set `CODEQL_PATH_TRANSFORMER` ourselves in some cases, so blank it out first for consistency - os.Setenv("CODEQL_PATH_TRANSFORMER", "") - os.Setenv("SEMMLE_PATH_TRANSFORMER", "") - // determine how to install dependencies and whether a GOPATH needs to be set up before // extraction workspaces := project.GetWorkspaceInfo(true) @@ -535,7 +532,21 @@ func installDependenciesAndBuild() { pt := createPathTransformerFile(paths.newdir) defer os.Remove(pt.Name()) - writePathTransformerFile(pt, paths.realSrc, paths.root, paths.newdir) + // We're about to create out own path transformer, so that paths containing the + // temporary GOPATH point to the right location. However, if there was already an + // incoming path transformer, the right location will be what _it_ wanted to transform + // paths to. + existingPathTransformer, err := srcarchive.LoadProjectLayoutFromEnv() + if err != nil { + log.Fatalf("Unable to load path transformer: %s.\n", err.Error()) + } + var realSrc string + if existingPathTransformer == nil { + realSrc = paths.realSrc + } else { + realSrc = existingPathTransformer.To + } + writePathTransformerFile(pt, realSrc, paths.newdir) setGopath(paths.root) } } diff --git a/go/extractor/srcarchive/projectlayout.go b/go/extractor/srcarchive/projectlayout.go index 6301755ef8a..0a628f38424 100644 --- a/go/extractor/srcarchive/projectlayout.go +++ b/go/extractor/srcarchive/projectlayout.go @@ -6,6 +6,8 @@ import ( "fmt" "os" "strings" + + "github.com/github/codeql-go/extractor/util" ) // ProjectLayout describes a very simple project layout rewriting paths starting @@ -16,7 +18,7 @@ import ( // # to // from// type ProjectLayout struct { - from, to string + From, To string } // normaliseSlashes adds an initial slash to `path` if there isn't one, and trims @@ -28,6 +30,25 @@ func normaliseSlashes(path string) string { return strings.TrimSuffix(path, "/") } +// LoadProjectLayoutFromEnv loads a project layout from the file referenced by the +// {CODEQL,SEMMLE}_PATH_TRANSFORMER environment variable. If neither env var is set, returns nil. If +// the file cannot be read or does not have the right format, it returns an error. +func LoadProjectLayoutFromEnv() (*ProjectLayout, error) { + pt := util.Getenv("CODEQL_PATH_TRANSFORMER", "SEMMLE_PATH_TRANSFORMER") + if pt == "" { + return nil, nil + } + ptf, err := os.Open(pt) + if err != nil { + return nil, err + } + projLayout, err := LoadProjectLayout(ptf) + if err != nil { + return nil, err + } + return projLayout, nil +} + // LoadProjectLayout loads a project layout from the given file, returning an error // if the file does not have the right format func LoadProjectLayout(file *os.File) (*ProjectLayout, error) { @@ -41,7 +62,7 @@ func LoadProjectLayout(file *os.File) (*ProjectLayout, error) { if !strings.HasPrefix(line, "#") { return nil, fmt.Errorf("first line of project layout should start with #, but got %s", line) } - res.to = normaliseSlashes(strings.TrimSpace(strings.TrimPrefix(line, "#"))) + res.To = normaliseSlashes(strings.TrimSpace(strings.TrimPrefix(line, "#"))) if !scanner.Scan() { return nil, errors.New("empty section in project-layout file") @@ -57,7 +78,7 @@ func LoadProjectLayout(file *os.File) (*ProjectLayout, error) { if strings.HasPrefix(line, "-") || strings.Contains(line, "*") || strings.Contains(line, "//") { return nil, errors.New("unsupported project-layout feature") } - res.from = normaliseSlashes(line) + res.From = normaliseSlashes(line) for scanner.Scan() { if strings.TrimSpace(scanner.Text()) != "" { @@ -71,11 +92,11 @@ func LoadProjectLayout(file *os.File) (*ProjectLayout, error) { // transformString transforms `str` as specified by the project layout: if it starts with the `from` // prefix, that prefix is relaced by `to`; otherwise the string is returned unchanged func (p *ProjectLayout) transformString(str string) string { - if str == p.from { - return p.to + if str == p.From { + return p.To } - if strings.HasPrefix(str, p.from+"/") { - return p.to + "/" + str[len(p.from)+1:] + if strings.HasPrefix(str, p.From+"/") { + return p.To + "/" + str[len(p.From)+1:] } return str } diff --git a/go/extractor/srcarchive/projectlayout_test.go b/go/extractor/srcarchive/projectlayout_test.go index 8fb17607cae..507ee183d9a 100644 --- a/go/extractor/srcarchive/projectlayout_test.go +++ b/go/extractor/srcarchive/projectlayout_test.go @@ -28,6 +28,32 @@ func mkProjectLayout(projectLayoutSource string, t *testing.T) (*ProjectLayout, return LoadProjectLayout(pt) } +func mkProjectLayoutFromEnv(projectLayoutSource string, t *testing.T) (*ProjectLayout, error) { + pt, err := os.CreateTemp("", "path-transformer-from-env") + if err != nil { + t.Fatalf("Unable to create temporary file for project layout: %s", err.Error()) + } + defer os.Remove(pt.Name()) + _, err = pt.WriteString(projectLayoutSource) + if err != nil { + t.Fatalf("Unable to write to temporary file for project layout: %s", err.Error()) + } + err = pt.Close() + if err != nil { + t.Fatalf("Unable to close path transformer file: %s.", err.Error()) + } + + pt, err = os.Open(pt.Name()) + if err != nil { + t.Fatalf("Unable to open path transformer file: %s.", err.Error()) + } + + os.Setenv("CODEQL_PATH_TRANSFORMER", pt.Name()) + defer os.Unsetenv("CODEQL_PATH_TRANSFORMER") + + return LoadProjectLayoutFromEnv() +} + func testTransformation(projectLayout *ProjectLayout, t *testing.T, path string, expected string) { actual := projectLayout.Transform(path) if actual != expected { @@ -35,16 +61,12 @@ func testTransformation(projectLayout *ProjectLayout, t *testing.T, path string, } } -func TestValidProjectLayout(t *testing.T) { - p, err := mkProjectLayout(` +const validProjectLayoutSource = ` # /opt/src /opt/src/root/src/org/repo// -`, t) - - if err != nil { - t.Fatalf("Error loading project layout: %s", err.Error()) - } +` +func testTransformationsForValidProjectLayout(p *ProjectLayout, t *testing.T) { testTransformation(p, t, "/opt/src/root/src/org/repo", "/opt/src") testTransformation(p, t, "/opt/src/root/src/org/repo/", "/opt/src/") testTransformation(p, t, "/opt/src/root/src/org/repo/main.go", "/opt/src/main.go") @@ -53,6 +75,26 @@ func TestValidProjectLayout(t *testing.T) { testTransformation(p, t, "opt/src/root/src/org/repo", "opt/src/root/src/org/repo") } +func TestValidProjectLayout(t *testing.T) { + p, err := mkProjectLayout(validProjectLayoutSource, t) + + if err != nil { + t.Fatalf("Error loading project layout: %s", err.Error()) + } + + testTransformationsForValidProjectLayout(p, t) +} + +func TestValidProjectLayoutFromEnv(t *testing.T) { + p, err := mkProjectLayoutFromEnv(validProjectLayoutSource, t) + + if err != nil { + t.Fatalf("Error loading project layout: %s", err.Error()) + } + + testTransformationsForValidProjectLayout(p, t) +} + func TestWindowsPaths(t *testing.T) { p, err := mkProjectLayout(` # /c:/virtual diff --git a/go/extractor/srcarchive/srcarchive.go b/go/extractor/srcarchive/srcarchive.go index eaaf867c607..f871a2fbca2 100644 --- a/go/extractor/srcarchive/srcarchive.go +++ b/go/extractor/srcarchive/srcarchive.go @@ -7,23 +7,16 @@ import ( "os" "path/filepath" "strings" - - "github.com/github/codeql-go/extractor/util" ) var pathTransformer *ProjectLayout func init() { - pt := util.Getenv("CODEQL_PATH_TRANSFORMER", "SEMMLE_PATH_TRANSFORMER") - if pt != "" { - ptf, err := os.Open(pt) - if err != nil { - log.Fatalf("Unable to open path transformer %s: %s.\n", pt, err.Error()) - } - pathTransformer, err = LoadProjectLayout(ptf) - if err != nil { - log.Fatalf("Unable to initialize path transformer: %s.\n", err.Error()) - } + pt, err := LoadProjectLayoutFromEnv() + if err == nil { + pathTransformer = pt + } else { + log.Fatalf("Unable to load path transformer: %s.\n", err.Error()) } }