hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #20623 from github/nickrolfe/go-extractor-overlay

Browse Source 
Go: basic overlay support
This commit is contained in:
Nick Rolfe
2025-11-12 14:56:25 +00:00
committed by GitHub
parent 085d3c85cd e5ba4143ff
commit 86465b36e0
25 changed files with 2628 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
go/extractor/cli/go-autobuilder/BUILD.bazel generated
View File

@@ -12,6 +12,7 @@ go_library(
"//go/extractor/autobuilder",
"//go/extractor/diagnostics",
"//go/extractor/project",
"//go/extractor/srcarchive",
"//go/extractor/toolchain",
"//go/extractor/util",
],

46
go/extractor/cli/go-autobuilder/go-autobuilder.go
View File

@@ -13,6 +13,7 @@ import (
"github.com/github/codeql-go/extractor/autobuilder"
"github.com/github/codeql-go/extractor/diagnostics"
"github.com/github/codeql-go/extractor/project"
"github.com/github/codeql-go/extractor/srcarchive"
"github.com/github/codeql-go/extractor/toolchain"
"github.com/github/codeql-go/extractor/util"
)
@@ -273,7 +274,7 @@ func createPathTransformerFile(newdir string) *os.File {
log.Fatalf("Failed to chdir into %s: %s\n", newdir, err.Error())
}
// set up SEMMLE_PATH_TRANSFORMER to ensure paths in the source archive and the snapshot
// set up CODEQL_PATH_TRANSFORMER to ensure paths in the source archive and the snapshot
// match the original source location, not the location we moved it to
pt, err := os.CreateTemp("", "path-transformer")
if err != nil {
@@ -283,7 +284,7 @@ func createPathTransformerFile(newdir string) *os.File {
}
// Writes the path transformer file
func writePathTransformerFile(pt *os.File, realSrc, root, newdir string) {
func writePathTransformerFile(pt *os.File, realSrc, newdir string) {
_, err := pt.WriteString("#" + realSrc + "\n" + newdir + "//\n")
if err != nil {
log.Fatalf("Unable to write path transformer file: %s.", err.Error())
@@ -292,9 +293,9 @@ func writePathTransformerFile(pt *os.File, realSrc, root, newdir string) {
if err != nil {
log.Fatalf("Unable to close path transformer file: %s.", err.Error())
}
err = os.Setenv("SEMMLE_PATH_TRANSFORMER", pt.Name())
err = os.Setenv("CODEQL_PATH_TRANSFORMER", pt.Name())
if err != nil {
log.Fatalf("Unable to set SEMMLE_PATH_TRANSFORMER environment variable: %s.\n", err.Error())
log.Fatalf("Unable to set CODEQL_PATH_TRANSFORMER environment variable: %s.\n", err.Error())
}
}
@@ -447,7 +448,7 @@ func installDependencies(workspace project.GoWorkspace) {
}
// Run the extractor.
func extract(workspace project.GoWorkspace) bool {
func extract(workspace project.GoWorkspace, sourceRoot string) bool {
extractor, err := util.GetExtractorPath()
if err != nil {
log.Fatalf("Could not determine path of extractor: %v.\n", err)
@@ -458,6 +459,12 @@ func extract(workspace project.GoWorkspace) bool {
extractorArgs = append(extractorArgs, workspace.ModMode.ArgsForGoVersion(toolchain.GetEnvGoSemVer())...)
}
if util.IsOverlayExtraction() {
// When we are extracting an overlay, pass the source root to the extractor so that it knows
// how to resolve the relative paths in the list of changed files.
extractorArgs = append(extractorArgs, "--source-root", sourceRoot)
}
if len(workspace.Modules) == 0 {
// There may be no modules if we are using e.g. Dep or Glide
extractorArgs = append(extractorArgs, "./...")
@@ -501,9 +508,6 @@ func installDependenciesAndBuild() {
srcdir := getSourceDir()
// we set `SEMMLE_PATH_TRANSFORMER` ourselves in some cases, so blank it out first for consistency
os.Setenv("SEMMLE_PATH_TRANSFORMER", "")
// determine how to install dependencies and whether a GOPATH needs to be set up before
// extraction
workspaces := project.GetWorkspaceInfo(true)
@@ -534,7 +538,21 @@ func installDependenciesAndBuild() {
pt := createPathTransformerFile(paths.newdir)
defer os.Remove(pt.Name())
writePathTransformerFile(pt, paths.realSrc, paths.root, paths.newdir)
// We're about to create out own path transformer, so that paths containing the
// temporary GOPATH point to the right location. However, if there was already an
// incoming path transformer, the right location will be what _it_ wanted to transform
// paths to.
existingPathTransformer, err := srcarchive.LoadProjectLayoutFromEnv()
if err != nil {
log.Fatalf("Unable to load path transformer: %s.\n", err.Error())
}
var realSrc string
if existingPathTransformer == nil {
realSrc = paths.realSrc
} else {
realSrc = existingPathTransformer.To
}
writePathTransformerFile(pt, realSrc, paths.newdir)
setGopath(paths.root)
}
}
@@ -575,6 +593,12 @@ func installDependenciesAndBuild() {
buildWithCustomCommands(inst)
}
// The autobuilder is invoked with its working directory set to the source directory.
sourceRoot, err := os.Getwd()
if err != nil {
log.Fatalf("Failed to get current working directory: %s\n", err.Error())
}
// Attempt to extract all workspaces; we will tolerate individual extraction failures here
for i, workspace := range workspaces {
if workspace.ModMode == project.ModVendor {
@@ -595,7 +619,7 @@ func installDependenciesAndBuild() {
}
}
workspaces[i].Extracted = extract(workspace)
workspaces[i].Extracted = extract(workspace, sourceRoot)
if !workspaces[i].Extracted {
unsuccessfulProjects = append(unsuccessfulProjects, workspace.BaseDir)
@@ -620,6 +644,8 @@ func installDependenciesAndBuild() {
} else {
log.Printf("Success: extraction succeeded for all %d discovered project(s).\n", len(workspaces))
}
util.WriteOverlayBaseMetadata()
}
func main() {

21
go/extractor/cli/go-extractor/go-extractor.go
View File

@@ -24,9 +24,10 @@ func usage() {
// extractTests is set (a) if we were manually commanded to extract tests via the relevant
// environment variable / extractor option, or (b) we're mimicking a `go test` command.
func parseFlags(args []string, mimic bool, extractTests bool) ([]string, []string, bool) {
func parseFlags(args []string, mimic bool, extractTests bool) ([]string, []string, bool, string) {
i := 0
buildFlags := []string{}
var sourceRoot string
for ; i < len(args) && strings.HasPrefix(args[i], "-"); i++ {
if args[i] == "--" {
i++
@@ -61,6 +62,18 @@ func parseFlags(args []string, mimic bool, extractTests bool) ([]string, []strin
} else {
log.Fatalf("--mimic requires an argument, e.g. --mimic go")
}
case "--source-root":
// The extractor can be called by the autobuilder with the working directory set to
// the directory containing the workspace we're extracting, and this may be a
// subdirectory of the actual source root. This argument lets us resolve paths that
// are relative to that source root, e.g. for the list of overlay changed files.
if i+1 < len(args) {
i++
sourceRoot = args[i]
log.Printf("Source root is %s", sourceRoot)
} else {
log.Fatalf("--source-root requires an argument, e.g. --source-root /path/to/root")
}
}
}
@@ -93,14 +106,14 @@ func parseFlags(args []string, mimic bool, extractTests bool) ([]string, []strin
cpuprofile = os.Getenv("CODEQL_EXTRACTOR_GO_CPU_PROFILE")
memprofile = os.Getenv("CODEQL_EXTRACTOR_GO_MEM_PROFILE")
return buildFlags, args[i:], extractTests
return buildFlags, args[i:], extractTests, sourceRoot
}
func main() {
util.SetLogLevel()
extractTestsDefault := os.Getenv("CODEQL_EXTRACTOR_GO_OPTION_EXTRACT_TESTS") == "true"
buildFlags, patterns, extractTests := parseFlags(os.Args[1:], false, extractTestsDefault)
buildFlags, patterns, extractTests, sourceRoot := parseFlags(os.Args[1:], false, extractTestsDefault)
if cpuprofile != "" {
f, err := os.Create(cpuprofile)
@@ -120,7 +133,7 @@ func main() {
}
log.Printf("Build flags: '%s'; patterns: '%s'\n", strings.Join(buildFlags, " "), strings.Join(patterns, " "))
err := extractor.ExtractWithFlags(buildFlags, patterns, extractTests)
err := extractor.ExtractWithFlags(buildFlags, patterns, extractTests, sourceRoot)
if err != nil {
errString := err.Error()
if strings.Contains(errString, "unexpected directory layout:") {

11
go/extractor/dbscheme/tables.go
View File

@@ -43,6 +43,17 @@ externalData(
snapshotDate(unique date snapshotDate : date ref);
sourceLocationPrefix(varchar(900) prefix : string ref);
/** Overlay support **/
databaseMetadata(
string metadataKey: string ref,
string value: string ref
);
overlayChangedFiles(
string path: string ref
);
`)
// Copied directly from the XML dbscheme

72
go/extractor/extractor.go
View File

@@ -11,6 +11,7 @@ import (
"go/types"
"io"
"log"
"log/slog"
"os"
"path/filepath"
"reflect"
@@ -58,16 +59,11 @@ func init() {
}
}
// Extract extracts the packages specified by the given patterns
func Extract(patterns []string) error {
return ExtractWithFlags(nil, patterns, false)
}
// ExtractWithFlags extracts the packages specified by the given patterns and build flags
func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool) error {
func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool, sourceRoot string) error {
startTime := time.Now()
extraction := NewExtraction(buildFlags, patterns)
extraction := NewExtraction(buildFlags, patterns, sourceRoot)
defer extraction.StatWriter.Close()
modEnabled := os.Getenv("GO111MODULE") != "off"
@@ -323,16 +319,17 @@ func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool)
type Extraction struct {
// A lock for preventing concurrent writes to maps and the stat trap writer, as they are not
// thread-safe
Lock sync.Mutex
LabelKey string
Label trap.Label
StatWriter *trap.Writer
WaitGroup sync.WaitGroup
GoroutineSem *semaphore
FdSem *semaphore
NextFileId int
FileInfo map[string]*FileInfo
SeenGoMods map[string]bool
Lock sync.Mutex
LabelKey string
Label trap.Label
StatWriter *trap.Writer
WaitGroup sync.WaitGroup
GoroutineSem *semaphore
FdSem *semaphore
NextFileId int
FileInfo map[string]*FileInfo
SeenGoMods map[string]bool
OverlayChanges map[string]bool
}
type FileInfo struct {
@@ -367,7 +364,7 @@ func (extraction *Extraction) GetNextErr(path string) int {
return res
}
func NewExtraction(buildFlags []string, patterns []string) *Extraction {
func NewExtraction(buildFlags []string, patterns []string, sourceRoot string) *Extraction {
hash := md5.New()
io.WriteString(hash, "go")
for _, buildFlag := range buildFlags {
@@ -379,6 +376,22 @@ func NewExtraction(buildFlags []string, patterns []string) *Extraction {
}
sum := hash.Sum(nil)
overlayChangeList := util.GetOverlayChanges(sourceRoot)
var overlayChanges map[string]bool
if overlayChangeList == nil {
overlayChanges = nil
} else {
overlayChanges = make(map[string]bool)
for _, changedFilePath := range overlayChangeList {
absPath, err := filepath.Abs(changedFilePath)
if err != nil {
log.Fatalf("Error resolving absolute path of overlay change %s: %s", changedFilePath, err.Error())
}
overlayChanges[absPath] = true
slog.Info("Overlay changed file", "path", absPath)
}
}
i := 0
var path string
// split compilation files into directories to avoid filling a single directory with too many files
@@ -438,10 +451,11 @@ func NewExtraction(buildFlags []string, patterns []string) *Extraction {
FdSem: newSemaphore(100),
// this semaphore is used to limit the number of goroutines spawned, so we
// don't run into memory issues
GoroutineSem: newSemaphore(MaxGoRoutines),
NextFileId: 0,
FileInfo: make(map[string]*FileInfo),
SeenGoMods: make(map[string]bool),
GoroutineSem: newSemaphore(MaxGoRoutines),
NextFileId: 0,
FileInfo: make(map[string]*FileInfo),
SeenGoMods: make(map[string]bool),
OverlayChanges: overlayChanges,
}
}
@@ -720,6 +734,16 @@ func (extraction *Extraction) extractFile(ast *ast.File, pkg *packages.Package)
return nil
}
path := normalizedPath(ast, fset)
// If we're extracting an overlay, we want to skip extraction of files that haven't changed.
// Since some files may be outside the source directory (e.g. files preprocessed by cgo) we
// can't easily know if they have changed (or came from source files that changed), so we always
// extract a file if it's not in the package directory.
if extraction.OverlayChanges != nil &&
!extraction.OverlayChanges[path] &&
strings.HasPrefix(path+string(filepath.Separator), pkg.Dir) {
slog.Info("Skipping unchanged file in overlay extraction", "path", path)
return nil
}
extraction.FdSem.acquire(3)
@@ -791,12 +815,12 @@ func (extraction *Extraction) extractFileInfo(tw *trap.Writer, file string, isDu
displayPath = rawPath
}
if i == len(components)-1 {
lbl := tw.Labeler.FileLabelFor(file)
lbl := tw.Labeler.FileLabelFor(path)
dbscheme.FilesTable.Emit(tw, lbl, displayPath)
dbscheme.ContainerParentTable.Emit(tw, parentLbl, lbl)
dbscheme.HasLocationTable.Emit(tw, lbl, emitLocation(tw, lbl, 0, 0, 0, 0))
extraction.Lock.Lock()
slbl := extraction.StatWriter.Labeler.FileLabelFor(file)
slbl := extraction.StatWriter.Labeler.FileLabelFor(path)
if !isDummy {
dbscheme.CompilationCompilingFilesTable.Emit(extraction.StatWriter, extraction.Label, extraction.GetFileIdx(file), slbl)
}

5
go/extractor/gomodextractor.go
View File

@@ -20,6 +20,11 @@ func (extraction *Extraction) extractGoMod(path string) error {
path = normPath
}
if extraction.OverlayChanges != nil && !extraction.OverlayChanges[path] {
// This go.mod did not change since the base was extracted
return nil
}
extraction.Lock.Lock()
if extraction.SeenGoMods[path] {
extraction.Lock.Unlock()

1
go/extractor/srcarchive/BUILD.bazel generated
View File

@@ -10,6 +10,7 @@ go_library(
],
importpath = "github.com/github/codeql-go/extractor/srcarchive",
visibility = ["//visibility:public"],
deps = ["//go/extractor/util"],
)
go_test(

35
go/extractor/srcarchive/projectlayout.go
View File

@@ -6,6 +6,8 @@ import (
"fmt"
"os"
"strings"
"github.com/github/codeql-go/extractor/util"
)
// ProjectLayout describes a very simple project layout rewriting paths starting
@@ -16,7 +18,7 @@ import (
// # to
// from//
type ProjectLayout struct {
from, to string
From, To string
}
// normaliseSlashes adds an initial slash to `path` if there isn't one, and trims
@@ -28,6 +30,25 @@ func normaliseSlashes(path string) string {
return strings.TrimSuffix(path, "/")
}
// LoadProjectLayoutFromEnv loads a project layout from the file referenced by the
// {CODEQL,SEMMLE}_PATH_TRANSFORMER environment variable. If neither env var is set, returns nil. If
// the file cannot be read or does not have the right format, it returns an error.
func LoadProjectLayoutFromEnv() (*ProjectLayout, error) {
pt := util.Getenv("CODEQL_PATH_TRANSFORMER", "SEMMLE_PATH_TRANSFORMER")
if pt == "" {
return nil, nil
}
ptf, err := os.Open(pt)
if err != nil {
return nil, err
}
projLayout, err := LoadProjectLayout(ptf)
if err != nil {
return nil, err
}
return projLayout, nil
}
// LoadProjectLayout loads a project layout from the given file, returning an error
// if the file does not have the right format
func LoadProjectLayout(file *os.File) (*ProjectLayout, error) {
@@ -41,7 +62,7 @@ func LoadProjectLayout(file *os.File) (*ProjectLayout, error) {
if !strings.HasPrefix(line, "#") {
return nil, fmt.Errorf("first line of project layout should start with #, but got %s", line)
}
res.to = normaliseSlashes(strings.TrimSpace(strings.TrimPrefix(line, "#")))
res.To = normaliseSlashes(strings.TrimSpace(strings.TrimPrefix(line, "#")))
if !scanner.Scan() {
return nil, errors.New("empty section in project-layout file")
@@ -57,7 +78,7 @@ func LoadProjectLayout(file *os.File) (*ProjectLayout, error) {
if strings.HasPrefix(line, "-") || strings.Contains(line, "*") || strings.Contains(line, "//") {
return nil, errors.New("unsupported project-layout feature")
}
res.from = normaliseSlashes(line)
res.From = normaliseSlashes(line)
for scanner.Scan() {
if strings.TrimSpace(scanner.Text()) != "" {
@@ -71,11 +92,11 @@ func LoadProjectLayout(file *os.File) (*ProjectLayout, error) {
// transformString transforms `str` as specified by the project layout: if it starts with the `from`
// prefix, that prefix is relaced by `to`; otherwise the string is returned unchanged
func (p *ProjectLayout) transformString(str string) string {
if str == p.from {
return p.to
if str == p.From {
return p.To
}
if strings.HasPrefix(str, p.from+"/") {
return p.to + "/" + str[len(p.from)+1:]
if strings.HasPrefix(str, p.From+"/") {
return p.To + "/" + str[len(p.From)+1:]
}
return str
}

56
go/extractor/srcarchive/projectlayout_test.go
View File

@@ -28,6 +28,32 @@ func mkProjectLayout(projectLayoutSource string, t *testing.T) (*ProjectLayout,
return LoadProjectLayout(pt)
}
func mkProjectLayoutFromEnv(projectLayoutSource string, t *testing.T) (*ProjectLayout, error) {
pt, err := os.CreateTemp("", "path-transformer-from-env")
if err != nil {
t.Fatalf("Unable to create temporary file for project layout: %s", err.Error())
}
defer os.Remove(pt.Name())
_, err = pt.WriteString(projectLayoutSource)
if err != nil {
t.Fatalf("Unable to write to temporary file for project layout: %s", err.Error())
}
err = pt.Close()
if err != nil {
t.Fatalf("Unable to close path transformer file: %s.", err.Error())
}
pt, err = os.Open(pt.Name())
if err != nil {
t.Fatalf("Unable to open path transformer file: %s.", err.Error())
}
os.Setenv("CODEQL_PATH_TRANSFORMER", pt.Name())
defer os.Unsetenv("CODEQL_PATH_TRANSFORMER")
return LoadProjectLayoutFromEnv()
}
func testTransformation(projectLayout *ProjectLayout, t *testing.T, path string, expected string) {
actual := projectLayout.Transform(path)
if actual != expected {
@@ -35,16 +61,12 @@ func testTransformation(projectLayout *ProjectLayout, t *testing.T, path string,
}
}
func TestValidProjectLayout(t *testing.T) {
p, err := mkProjectLayout(`
const validProjectLayoutSource = `
# /opt/src
/opt/src/root/src/org/repo//
`, t)
if err != nil {
t.Fatalf("Error loading project layout: %s", err.Error())
}
`
func testTransformationsForValidProjectLayout(p *ProjectLayout, t *testing.T) {
testTransformation(p, t, "/opt/src/root/src/org/repo", "/opt/src")
testTransformation(p, t, "/opt/src/root/src/org/repo/", "/opt/src/")
testTransformation(p, t, "/opt/src/root/src/org/repo/main.go", "/opt/src/main.go")
@@ -53,6 +75,26 @@ func TestValidProjectLayout(t *testing.T) {
testTransformation(p, t, "opt/src/root/src/org/repo", "opt/src/root/src/org/repo")
}
func TestValidProjectLayout(t *testing.T) {
p, err := mkProjectLayout(validProjectLayoutSource, t)
if err != nil {
t.Fatalf("Error loading project layout: %s", err.Error())
}
testTransformationsForValidProjectLayout(p, t)
}
func TestValidProjectLayoutFromEnv(t *testing.T) {
p, err := mkProjectLayoutFromEnv(validProjectLayoutSource, t)
if err != nil {
t.Fatalf("Error loading project layout: %s", err.Error())
}
testTransformationsForValidProjectLayout(p, t)
}
func TestWindowsPaths(t *testing.T) {
p, err := mkProjectLayout(`
# /c:/virtual

19
go/extractor/srcarchive/srcarchive.go
View File

@@ -4,6 +4,7 @@ import (
"errors"
"io"
"log"
"log/slog"
"os"
"path/filepath"
"strings"
@@ -12,16 +13,14 @@ import (
var pathTransformer *ProjectLayout
func init() {
pt := os.Getenv("SEMMLE_PATH_TRANSFORMER")
if pt != "" {
ptf, err := os.Open(pt)
if err != nil {
log.Fatalf("Unable to open path transformer %s: %s.\n", pt, err.Error())
}
pathTransformer, err = LoadProjectLayout(ptf)
if err != nil {
log.Fatalf("Unable to initialize path transformer: %s.\n", err.Error())
pt, err := LoadProjectLayoutFromEnv()
if err == nil {
pathTransformer = pt
if pathTransformer != nil {
slog.Info("Loaded path transformer", "from", pathTransformer.From, "to", pathTransformer.To)
}
} else {
log.Fatalf("Unable to load path transformer: %s.\n", err.Error())
}
}
@@ -69,7 +68,7 @@ func srcArchive() (string, error) {
return srcArchive, nil
}
// TransformPath applies the transformations specified by `SEMMLE_PATH_TRANSFORMER` (if any) to the
// TransformPath applies the transformations specified by `CODEQL_PATH_TRANSFORMER` (if any) to the
// given path
func TransformPath(path string) string {
if pathTransformer != nil {

6
go/extractor/trap/labels.go
View File

@@ -3,7 +3,9 @@ package trap
import (
"fmt"
"go/types"
"path/filepath"
"github.com/github/codeql-go/extractor/srcarchive"
"github.com/github/codeql-go/extractor/util"
)
@@ -67,14 +69,14 @@ func (l *Labeler) GlobalID(key string) Label {
// FileLabel returns the label for a file with path `path`.
func (l *Labeler) FileLabel() Label {
if l.fileLabel == InvalidLabel {
l.fileLabel = l.FileLabelFor(l.tw.path)
l.fileLabel = l.FileLabelFor(srcarchive.TransformPath(l.tw.path))
}
return l.fileLabel
}
// FileLabelFor returns the label for the file for which the trap writer `tw` is associated
func (l *Labeler) FileLabelFor(path string) Label {
return l.GlobalID(util.EscapeTrapSpecialChars(path) + ";sourcefile")
return l.GlobalID(util.EscapeTrapSpecialChars(filepath.ToSlash(path)) + ";sourcefile")
}
// LocalID associates a label with the given AST node `nd` and returns it

1
go/extractor/util/BUILD.bazel generated
View File

@@ -7,6 +7,7 @@ go_library(
srcs = [
"extractvendordirs.go",
"logging.go",
"overlays.go",
"registryproxy.go",
"semver.go",
"util.go",

70
go/extractor/util/overlays.go Normal file
View File

@@ -0,0 +1,70 @@
package util
import (
"encoding/json"
"log"
"os"
"path/filepath"
)
func IsOverlayExtraction() bool {
_, present := os.LookupEnv("CODEQL_EXTRACTOR_GO_OVERLAY_METADATA_IN")
return present
}
// If the relevant environment variable is set, indicating that we are extracting an overlay
// database, GetOverlayChanges returns the list of relative paths of files that have changed (or
// been deleted). Otherwise, it returns `nil`.
func GetOverlayChanges(sourceRoot string) []string {
if overlayChangesJsonPath, present := os.LookupEnv("CODEQL_EXTRACTOR_GO_OVERLAY_CHANGES"); present {
log.Printf("Reading overlay changes from: %s", overlayChangesJsonPath)
file, err := os.Open(overlayChangesJsonPath)
if err != nil {
log.Fatalf("Failed to open overlay changes JSON file: %s", err)
}
defer file.Close()
var overlayData struct {
Changes []string `json:"changes"`
}
decoder := json.NewDecoder(file)
if err := decoder.Decode(&overlayData); err != nil {
log.Fatalf("Failed to decode overlay changes JSON file: %s", err)
}
absPaths := make([]string, 0, len(overlayData.Changes))
if sourceRoot == "" {
// This shouldn't happen, because it implies the extractor was invoked in some way other
// than from the autobuilder. However, we'll only attempt to build an overlay if there
// exists an overlay _base_, and only the autobuilder writes the metadata file that
// ensures a database is created as an overlay-base.
log.Fatalf("Extractor is running in overlay mode, but --source-root was not provided")
}
for _, relPath := range overlayData.Changes {
absPaths = append(absPaths, filepath.Clean(sourceRoot+"/"+relPath))
}
return absPaths
} else {
return nil
}
}
// WriteOverlayBaseMetadata creates an empty metadata file if we are extracting an overlay base;
// otherwise, it does nothing.
func WriteOverlayBaseMetadata() {
if metadataPath, present := os.LookupEnv("CODEQL_EXTRACTOR_GO_OVERLAY_BASE_METADATA_OUT"); present {
log.Printf("Writing overlay base metadata to: %s", metadataPath)
// In principle, we could store some metadata here and read it back when extracting the
// overlay. For now, we don't need to store anything, but the CLI still requires us to write
// something, so just create an empty file.
file, err := os.Create(metadataPath)
if err != nil {
log.Fatalf("Failed to create overlay base metadata file: %s", err)
}
file.Close()
}
}