hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Go: merge with incoming path transformer when setting GOPATH

Browse Source
This commit is contained in:
Nick Rolfe
2025-09-03 16:36:28 +01:00
parent 99236f7877
commit aff874e835
5 changed files with 100 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
go/extractor/cli/go-autobuilder/BUILD.bazel generated
View File

@@ -12,6 +12,7 @@ go_library(
"//go/extractor/autobuilder",
"//go/extractor/diagnostics",
"//go/extractor/project",
"//go/extractor/srcarchive",
"//go/extractor/toolchain",
"//go/extractor/util",
],

23
go/extractor/cli/go-autobuilder/go-autobuilder.go
View File

@@ -13,6 +13,7 @@ import (
"github.com/github/codeql-go/extractor/autobuilder"
"github.com/github/codeql-go/extractor/diagnostics"
"github.com/github/codeql-go/extractor/project"
"github.com/github/codeql-go/extractor/srcarchive"
"github.com/github/codeql-go/extractor/toolchain"
"github.com/github/codeql-go/extractor/util"
)
@@ -283,7 +284,7 @@ func createPathTransformerFile(newdir string) *os.File {
}
// Writes the path transformer file
func writePathTransformerFile(pt *os.File, realSrc, root, newdir string) {
func writePathTransformerFile(pt *os.File, realSrc, newdir string) {
_, err := pt.WriteString("#" + realSrc + "\n" + newdir + "//\n")
if err != nil {
log.Fatalf("Unable to write path transformer file: %s.", err.Error())
@@ -501,10 +502,6 @@ func installDependenciesAndBuild() {
srcdir := getSourceDir()
// we set `CODEQL_PATH_TRANSFORMER` ourselves in some cases, so blank it out first for consistency
os.Setenv("CODEQL_PATH_TRANSFORMER", "")
os.Setenv("SEMMLE_PATH_TRANSFORMER", "")
// determine how to install dependencies and whether a GOPATH needs to be set up before
// extraction
workspaces := project.GetWorkspaceInfo(true)
@@ -535,7 +532,21 @@ func installDependenciesAndBuild() {
pt := createPathTransformerFile(paths.newdir)
defer os.Remove(pt.Name())
writePathTransformerFile(pt, paths.realSrc, paths.root, paths.newdir)
// We're about to create out own path transformer, so that paths containing the
// temporary GOPATH point to the right location. However, if there was already an
// incoming path transformer, the right location will be what _it_ wanted to transform
// paths to.
existingPathTransformer, err := srcarchive.LoadProjectLayoutFromEnv()
if err != nil {
log.Fatalf("Unable to load path transformer: %s.\n", err.Error())
}
var realSrc string
if existingPathTransformer == nil {
realSrc = paths.realSrc
} else {
realSrc = existingPathTransformer.To
}
writePathTransformerFile(pt, realSrc, paths.newdir)
setGopath(paths.root)
}
}

35
go/extractor/srcarchive/projectlayout.go
View File

@@ -6,6 +6,8 @@ import (
"fmt"
"os"
"strings"
"github.com/github/codeql-go/extractor/util"
)
// ProjectLayout describes a very simple project layout rewriting paths starting
@@ -16,7 +18,7 @@ import (
// # to
// from//
type ProjectLayout struct {
from, to string
From, To string
}
// normaliseSlashes adds an initial slash to `path` if there isn't one, and trims
@@ -28,6 +30,25 @@ func normaliseSlashes(path string) string {
return strings.TrimSuffix(path, "/")
}
// LoadProjectLayoutFromEnv loads a project layout from the file referenced by the
// {CODEQL,SEMMLE}_PATH_TRANSFORMER environment variable. If neither env var is set, returns nil. If
// the file cannot be read or does not have the right format, it returns an error.
func LoadProjectLayoutFromEnv() (*ProjectLayout, error) {
pt := util.Getenv("CODEQL_PATH_TRANSFORMER", "SEMMLE_PATH_TRANSFORMER")
if pt == "" {
return nil, nil
}
ptf, err := os.Open(pt)
if err != nil {
return nil, err
}
projLayout, err := LoadProjectLayout(ptf)
if err != nil {
return nil, err
}
return projLayout, nil
}
// LoadProjectLayout loads a project layout from the given file, returning an error
// if the file does not have the right format
func LoadProjectLayout(file *os.File) (*ProjectLayout, error) {
@@ -41,7 +62,7 @@ func LoadProjectLayout(file *os.File) (*ProjectLayout, error) {
if !strings.HasPrefix(line, "#") {
return nil, fmt.Errorf("first line of project layout should start with #, but got %s", line)
}
res.to = normaliseSlashes(strings.TrimSpace(strings.TrimPrefix(line, "#")))
res.To = normaliseSlashes(strings.TrimSpace(strings.TrimPrefix(line, "#")))
if !scanner.Scan() {
return nil, errors.New("empty section in project-layout file")
@@ -57,7 +78,7 @@ func LoadProjectLayout(file *os.File) (*ProjectLayout, error) {
if strings.HasPrefix(line, "-") || strings.Contains(line, "*") || strings.Contains(line, "//") {
return nil, errors.New("unsupported project-layout feature")
}
res.from = normaliseSlashes(line)
res.From = normaliseSlashes(line)
for scanner.Scan() {
if strings.TrimSpace(scanner.Text()) != "" {
@@ -71,11 +92,11 @@ func LoadProjectLayout(file *os.File) (*ProjectLayout, error) {
// transformString transforms `str` as specified by the project layout: if it starts with the `from`
// prefix, that prefix is relaced by `to`; otherwise the string is returned unchanged
func (p *ProjectLayout) transformString(str string) string {
if str == p.from {
return p.to
if str == p.From {
return p.To
}
if strings.HasPrefix(str, p.from+"/") {
return p.to + "/" + str[len(p.from)+1:]
if strings.HasPrefix(str, p.From+"/") {
return p.To + "/" + str[len(p.From)+1:]
}
return str
}

56
go/extractor/srcarchive/projectlayout_test.go
View File

@@ -28,6 +28,32 @@ func mkProjectLayout(projectLayoutSource string, t *testing.T) (*ProjectLayout,
return LoadProjectLayout(pt)
}
func mkProjectLayoutFromEnv(projectLayoutSource string, t *testing.T) (*ProjectLayout, error) {
pt, err := os.CreateTemp("", "path-transformer-from-env")
if err != nil {
t.Fatalf("Unable to create temporary file for project layout: %s", err.Error())
}
defer os.Remove(pt.Name())
_, err = pt.WriteString(projectLayoutSource)
if err != nil {
t.Fatalf("Unable to write to temporary file for project layout: %s", err.Error())
}
err = pt.Close()
if err != nil {
t.Fatalf("Unable to close path transformer file: %s.", err.Error())
}
pt, err = os.Open(pt.Name())
if err != nil {
t.Fatalf("Unable to open path transformer file: %s.", err.Error())
}
os.Setenv("CODEQL_PATH_TRANSFORMER", pt.Name())
defer os.Unsetenv("CODEQL_PATH_TRANSFORMER")
return LoadProjectLayoutFromEnv()
}
func testTransformation(projectLayout *ProjectLayout, t *testing.T, path string, expected string) {
actual := projectLayout.Transform(path)
if actual != expected {
@@ -35,16 +61,12 @@ func testTransformation(projectLayout *ProjectLayout, t *testing.T, path string,
}
}
func TestValidProjectLayout(t *testing.T) {
p, err := mkProjectLayout(`
const validProjectLayoutSource = `
# /opt/src
/opt/src/root/src/org/repo//
`, t)
if err != nil {
t.Fatalf("Error loading project layout: %s", err.Error())
}
`
func testTransformationsForValidProjectLayout(p *ProjectLayout, t *testing.T) {
testTransformation(p, t, "/opt/src/root/src/org/repo", "/opt/src")
testTransformation(p, t, "/opt/src/root/src/org/repo/", "/opt/src/")
testTransformation(p, t, "/opt/src/root/src/org/repo/main.go", "/opt/src/main.go")
@@ -53,6 +75,26 @@ func TestValidProjectLayout(t *testing.T) {
testTransformation(p, t, "opt/src/root/src/org/repo", "opt/src/root/src/org/repo")
}
func TestValidProjectLayout(t *testing.T) {
p, err := mkProjectLayout(validProjectLayoutSource, t)
if err != nil {
t.Fatalf("Error loading project layout: %s", err.Error())
}
testTransformationsForValidProjectLayout(p, t)
}
func TestValidProjectLayoutFromEnv(t *testing.T) {
p, err := mkProjectLayoutFromEnv(validProjectLayoutSource, t)
if err != nil {
t.Fatalf("Error loading project layout: %s", err.Error())
}
testTransformationsForValidProjectLayout(p, t)
}
func TestWindowsPaths(t *testing.T) {
p, err := mkProjectLayout(`
# /c:/virtual

17
go/extractor/srcarchive/srcarchive.go
View File

@@ -7,23 +7,16 @@ import (
"os"
"path/filepath"
"strings"
"github.com/github/codeql-go/extractor/util"
)
var pathTransformer *ProjectLayout
func init() {
pt := util.Getenv("CODEQL_PATH_TRANSFORMER", "SEMMLE_PATH_TRANSFORMER")
if pt != "" {
ptf, err := os.Open(pt)
if err != nil {
log.Fatalf("Unable to open path transformer %s: %s.\n", pt, err.Error())
}
pathTransformer, err = LoadProjectLayout(ptf)
if err != nil {
log.Fatalf("Unable to initialize path transformer: %s.\n", err.Error())
}
pt, err := LoadProjectLayoutFromEnv()
if err == nil {
pathTransformer = pt
} else {
log.Fatalf("Unable to load path transformer: %s.\n", err.Error())
}
}