hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-01-29 06:12:58 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #324 from sauyon/tracing

Browse Source 
Build tracing
This commit is contained in:
Chris Smowton
2020-10-15 11:27:34 +01:00
committed by GitHub
parent 4746789fe8 e5afd1dcb6
commit 2b07e6a0f4
13 changed files with 266 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@@ -14,11 +14,11 @@ CODEQL_PLATFORM = osx64
endif
endif
CODEQL_TOOLS = $(addprefix codeql-tools/,autobuild.cmd autobuild.sh index.cmd index.sh)
CODEQL_TOOLS = $(addprefix codeql-tools/,autobuild.cmd autobuild.sh index.cmd index.sh linux64 osx64 win64)
EXTRACTOR_PACK_OUT = build/codeql-extractor-go
BINARIES = go-extractor go-tokenizer go-autobuilder go-bootstrap go-gen-dbscheme
BINARIES = go-extractor go-tokenizer go-autobuilder go-build-runner go-bootstrap go-gen-dbscheme
.PHONY: tools tools-codeql tools-codeql-full clean autoformat \
tools-linux64 tools-osx64 tools-win64 check-formatting

5
change-notes/2020-06-11-build-tracing.md Normal file
View File

@@ -0,0 +1,5 @@
lgtm,codescanning
* The Go extractor now supports build tracing, allowing users to supply a build command when
creating databases with the CodeQL CLI or via configuration. It currently only supports projects
that use Go modules. To opt-in, set the environment variable `CODEQL_EXTRACTOR_GO_BUILD_TRACING`
to `on`, or supply a build command.

7
codeql-tools/autobuild.cmd
View File

@@ -4,7 +4,12 @@ SETLOCAL EnableDelayedExpansion
rem Some legacy environment variables for the autobuilder.
set LGTM_SRC=%CD%
type NUL && "%CODEQL_EXTRACTOR_GO_ROOT%/tools/%CODEQL_PLATFORM%/go-autobuilder.exe"
if "%CODEQL_EXTRACTOR_GO_BUILD_TRACING%"=="on" (
echo "Tracing enabled"
type NUL && "%CODEQL_EXTRACTOR_GO_ROOT%/tools/%CODEQL_PLATFORM%/go-build-runner.exe"
) else (
type NUL && "%CODEQL_EXTRACTOR_GO_ROOT%/tools/%CODEQL_PLATFORM%/go-autobuilder.exe"
)
exit /b %ERRORLEVEL%
ENDLOCAL

7
codeql-tools/autobuild.sh
View File

@@ -11,4 +11,9 @@ fi
LGTM_SRC="$(pwd)"
export LGTM_SRC
"$CODEQL_EXTRACTOR_GO_ROOT/tools/$CODEQL_PLATFORM/go-autobuilder"
if [ "${CODEQL_EXTRACTOR_GO_BUILD_TRACING:-}" == "on" ]; then
echo "Tracing enabled"
"$CODEQL_EXTRACTOR_GO_ROOT/tools/$CODEQL_PLATFORM/go-build-runner"
else
"$CODEQL_EXTRACTOR_GO_ROOT/tools/$CODEQL_PLATFORM/go-autobuilder"
fi

7
codeql-tools/linux64/compiler-tracing.spec Normal file
View File

@@ -0,0 +1,7 @@
**/go-autobuilder:
order compiler
trace no
**/go:
invoke ${config_dir}/go-extractor
prepend --mimic
prepend "${compiler}"

22
codeql-tools/osx64/compiler-tracing.spec Normal file
View File

@@ -0,0 +1,22 @@
**/go-autobuilder:
order compiler
trace no
**/go:
invoke ${config_dir}/go-extractor
prepend --mimic
prepend "${compiler}"
/usr/bin/codesign:
replace yes
invoke /usr/bin/env
prepend /usr/bin/codesign
trace no
/usr/bin/pkill:
replace yes
invoke /usr/bin/env
prepend /usr/bin/pkill
trace no
/usr/bin/pgrep:
replace yes
invoke /usr/bin/env
prepend /usr/bin/pgrep
trace no

7
codeql-tools/win64/compiler-tracing.spec Normal file
View File

@@ -0,0 +1,7 @@
**/go-autobuilder.exe:
order compiler
trace no
**/go.exe:
invoke ${config_dir}/go-extractor.exe
prepend --mimic
prepend "${compiler}"

81
extractor/autobuilder/autobuilder.go Normal file
View File

@@ -0,0 +1,81 @@
// Package autobuilder implements a simple system that attempts to run build commands for common
// build frameworks, if the relevant files exist.
package autobuilder
import (
"log"
"os"
"os/exec"
"github.com/github/codeql-go/extractor/util"
)
// CheckExtracted sets whether the autobuilder should check whether source files have been extracted
// to the CodeQL source directory as well as whether the build command executed successfully.
var CheckExtracted = false
// checkEmpty checks whether a directory either doesn't exist or is empty.
func checkEmpty(dir string) (bool, error) {
if !util.DirExists(dir) {
return true, nil
}
d, err := os.Open(dir)
if err != nil {
return false, err
}
defer d.Close()
names, err := d.Readdirnames(-1)
if err != nil {
return false, err
}
return len(names) == 0, nil
}
// checkExtractorRun checks whether the CodeQL Go extractor has run, by checking if the source
// archive directory is empty or not.
func checkExtractorRun() bool {
srcDir := os.Getenv("CODEQL_EXTRACTOR_GO_SOURCE_ARCHIVE_DIR")
if srcDir != "" {
empty, err := checkEmpty(srcDir)
if err != nil {
log.Fatalf("Unable to read source archive directory %s.", srcDir)
}
if empty {
log.Printf("No Go code seen; continuing to try other builds.")
return false
}
return true
} else {
log.Fatalf("No source directory set.\nThis binary should not be run manually; instead, use the CodeQL CLI or VSCode extension. See https://securitylab.github.com/tools/codeql.")
return false
}
}
// tryBuildIfExists tries to run the command `cmd args...` if the file `buildFile` exists and is not
// a directory. Returns true if the command was successful and false if not.
func tryBuildIfExists(buildFile, cmd string, args ...string) bool {
if util.FileExists(buildFile) {
log.Printf("%s found.\n", buildFile)
return tryBuild(cmd, args...)
}
return false
}
// tryBuild tries to run `cmd args...`, returning true if successful and false if not.
func tryBuild(cmd string, args ...string) bool {
log.Printf("Trying build command %s %v", cmd, args)
res := util.RunCmd(exec.Command(cmd, args...))
return res && (!CheckExtracted || checkExtractorRun())
}
// Autobuild attempts to detect build system and run the corresponding command.
func Autobuild() bool {
return tryBuildIfExists("Makefile", "make") ||
tryBuildIfExists("makefile", "make") ||
tryBuildIfExists("GNUmakefile", "make") ||
tryBuildIfExists("build.ninja", "ninja") ||
tryBuildIfExists("build", "./build") ||
tryBuildIfExists("build.sh", "./build.sh")
}

37
extractor/cli/go-autobuilder/go-autobuilder.go
View File

@@ -13,6 +13,7 @@ import (
"runtime"
"strings"
"github.com/github/codeql-go/extractor/autobuilder"
"github.com/github/codeql-go/extractor/util"
)
@@ -68,29 +69,10 @@ func getEnvGoSemVer() string {
return "v" + goVersion[2:]
}
func run(cmd *exec.Cmd) bool {
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
in, _ := cmd.StdinPipe()
err := cmd.Start()
if err != nil {
log.Printf("Running %s failed, continuing anyway: %s\n", cmd.Path, err.Error())
return false
}
in.Close()
err = cmd.Wait()
if err != nil {
log.Printf("Running %s failed, continuing anyway: %s\n", cmd.Path, err.Error())
return false
}
return true
}
func tryBuild(buildFile, cmd string, args ...string) bool {
if util.FileExists(buildFile) {
log.Printf("%s found, running %s\n", buildFile, cmd)
return run(exec.Command(cmd, args...))
return util.RunCmd(exec.Command(cmd, args...))
}
return false
}
@@ -209,7 +191,7 @@ func (m ModMode) argsForGoVersion(version string) []string {
// addVersionToMod add a go version directive, e.g. `go 1.14` to a `go.mod` file.
func addVersionToMod(goMod []byte, version string) bool {
cmd := exec.Command("go", "mod", "edit", "-go="+version)
return run(cmd)
return util.RunCmd(cmd)
}
// checkVendor tests to see whether a vendor directory is inconsistent according to the go frontend
@@ -422,13 +404,8 @@ func main() {
inst := util.Getenv("CODEQL_EXTRACTOR_GO_BUILD_COMMAND", "LGTM_INDEX_BUILD_COMMAND")
shouldInstallDependencies := false
if inst == "" {
// if there is a build file, run the corresponding build tool
buildSucceeded := tryBuild("Makefile", "make") ||
tryBuild("makefile", "make") ||
tryBuild("GNUmakefile", "make") ||
tryBuild("build.ninja", "ninja") ||
tryBuild("build", "./build") ||
tryBuild("build.sh", "./build.sh")
// try to build the project
buildSucceeded := autobuilder.Autobuild()
if !buildSucceeded {
// Build failed; we'll try to install dependencies ourselves
@@ -464,7 +441,7 @@ func main() {
}
os.Chmod(script.Name(), 0700)
log.Println("Installing dependencies using custom build command.")
run(exec.Command(script.Name()))
util.RunCmd(exec.Command(script.Name()))
}
if modMode == ModVendor {
@@ -525,7 +502,7 @@ func main() {
install = exec.Command("go", "get", "-v", "./...")
log.Println("Installing dependencies using `go get -v ./...`.")
}
run(install)
util.RunCmd(install)
}
}

36
extractor/cli/go-build-runner/go-build-runner.go Normal file
View File

@@ -0,0 +1,36 @@
package main
import (
"github.com/github/codeql-go/extractor/util"
"log"
"os"
"os/exec"
"path/filepath"
"runtime"
"github.com/github/codeql-go/extractor/autobuilder"
)
func main() {
// check if a build command has successfully extracted something
autobuilder.CheckExtracted = true
if autobuilder.Autobuild() {
return
}
// if the autobuilder fails, invoke the extractor manually
// we cannot simply call `go build` here, because the tracer is not able to trace calls made by
// this binary
log.Printf("No build commands succeeded, falling back to go build ./...")
mypath, err := os.Executable()
if err != nil {
log.Fatalf("Could not determine path of extractor: %v.\n", err)
}
extractor := filepath.Join(filepath.Dir(mypath), "go-extractor")
if runtime.GOOS == "windows" {
extractor = extractor + ".exe"
}
util.RunCmd(exec.Command(extractor, "./..."))
}

68
extractor/cli/go-extractor/go-extractor.go
View File

@@ -20,23 +20,70 @@ func usage() {
fmt.Fprintf(os.Stderr, "--help Print this help.\n")
}
func parseFlags(args []string) ([]string, []string) {
func parseFlags(args []string, mimic bool) ([]string, []string) {
i := 0
buildFlags := []string{}
for i < len(args) && strings.HasPrefix(args[i], "-") {
for ; i < len(args) && strings.HasPrefix(args[i], "-"); i++ {
if args[i] == "--" {
i++
break
}
if args[i] == "--help" {
usage()
os.Exit(0)
} else {
buildFlags = append(buildFlags, args[i])
if !mimic {
// we're not in mimic mode, try to parse our arguments
switch args[i] {
case "--help":
usage()
os.Exit(0)
case "--mimic":
if i+1 < len(args) {
i++
compiler := args[i]
log.Printf("Compiler: %s", compiler)
if i+1 < len(args) {
i++
command := args[i]
if command == "build" || command == "install" || command == "run" {
log.Printf("Intercepting build")
return parseFlags(args[i+1:], true)
} else {
log.Printf("Non-build command '%s'; skipping", strings.Join(args[1:], " "))
os.Exit(0)
}
} else {
log.Printf("Non-build command '%s'; skipping", strings.Join(args[1:], " "))
os.Exit(0)
}
} else {
log.Fatalf("--mimic requires an argument, e.g. --mimic go")
}
}
}
i++
// parse go build flags
switch args[i] {
// skip `-o output` and `-i`, if applicable
case "-o":
if i+1 < len(args) {
i++
}
case "-i":
case "-p", "-asmflags", "-buildmode", "-compiler", "-gccgoflags", "-gcflags", "-installsuffix",
"-ldflags", "-mod", "-modfile", "-pkgdir", "-tags", "-toolexec":
if i+1 < len(args) {
buildFlags = append(buildFlags, args[i], args[i+1])
i++
} else {
buildFlags = append(buildFlags, args[i])
}
default:
if strings.HasPrefix(args[i], "-") {
buildFlags = append(buildFlags, args[i])
} else {
// stop parsing if the argument is not a flag (and so is positional)
break
}
}
}
cpuprofile = os.Getenv("CODEQL_EXTRACTOR_GO_CPU_PROFILE")
@@ -46,7 +93,7 @@ func parseFlags(args []string) ([]string, []string) {
}
func main() {
buildFlags, patterns := parseFlags(os.Args[1:])
buildFlags, patterns := parseFlags(os.Args[1:], false)
if cpuprofile != "" {
f, err := os.Create(cpuprofile)
@@ -63,9 +110,10 @@ func main() {
if len(patterns) == 0 {
log.Println("Nothing to extract.")
} else {
log.Printf("Build flags: '%s'; patterns: '%s'\n", strings.Join(buildFlags, " "), strings.Join(patterns, " "))
err := extractor.ExtractWithFlags(buildFlags, patterns)
if err != nil {
log.Fatal(err)
log.Fatalf("Error running go tooling: %s\n", err.Error())
}
}

1
extractor/extractor.go
View File

@@ -122,6 +122,7 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
log.Fatalf("Unable to get a source directory for input package %s.", pkg.PkgPath)
}
wantedRoots[pkgRoots[pkg.PkgPath]] = true
wantedRoots[pkgDirs[pkg.PkgPath]] = true
}
log.Println("Done processing dependencies.")

30
extractor/util/util.go
View File

@@ -28,9 +28,14 @@ func Getenv(key string, aliases ...string) string {
// runGoList is a helper function for running go list with format `format` and flags `flags` on
// package `pkgpath`.
func runGoList(format string, pkgpath string, flags ...string) (string, error) {
return runGoListWithEnv(format, pkgpath, nil, flags...)
}
func runGoListWithEnv(format string, pkgpath string, additionalEnv []string, flags ...string) (string, error) {
args := append([]string{"list", "-e", "-f", format}, flags...)
args = append(args, pkgpath)
cmd := exec.Command("go", args...)
cmd.Env = append(os.Environ(), additionalEnv...)
out, err := cmd.Output()
if err != nil {
@@ -48,13 +53,15 @@ func runGoList(format string, pkgpath string, flags ...string) (string, error) {
// GetModDir gets the absolute directory of the module containing the package with path
// `pkgpath`. It passes the `go list` the flags specified by `flags`.
func GetModDir(pkgpath string, flags ...string) string {
mod, err := runGoList("{{.Module}}", pkgpath, flags...)
// enable module mode so that we can find a module root if it exists, even if go module support is
// disabled by a build
mod, err := runGoListWithEnv("{{.Module}}", pkgpath, []string{"GO111MODULE=on"}, flags...)
if err != nil || mod == "<nil>" {
// if the command errors or modules aren't being used, return the empty string
return ""
}
modDir, err := runGoList("{{.Module.Dir}}", pkgpath, flags...)
modDir, err := runGoListWithEnv("{{.Module.Dir}}", pkgpath, []string{"GO111MODULE=on"}, flags...)
if err != nil {
return ""
}
@@ -100,3 +107,22 @@ func DirExists(filename string) bool {
}
return err == nil && info.IsDir()
}
func RunCmd(cmd *exec.Cmd) bool {
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
in, _ := cmd.StdinPipe()
err := cmd.Start()
if err != nil {
log.Printf("Running %s failed, continuing anyway: %s\n", cmd.Path, err.Error())
return false
}
in.Close()
err = cmd.Wait()
if err != nil {
log.Printf("Running %s failed, continuing anyway: %s\n", cmd.Path, err.Error())
return false
}
return true
}