From c2c88b083553c49a75859d88450a7f0175334644 Mon Sep 17 00:00:00 2001 From: Chris Smowton Date: Thu, 25 Mar 2021 11:24:39 +0000 Subject: [PATCH 1/3] Unify two implementations of GetExtractorPath This retains both their features: * The new util.go one cached its result. * The old go-autobuilder.go one worked under ODASA, where CODEQL_GO_EXTRACTOR_ROOT is unset but os.Executable is a useful substitute. --- .../cli/go-autobuilder/go-autobuilder.go | 48 +------------- extractor/extractor.go | 6 +- extractor/util/util.go | 63 ++++++++++++++----- 3 files changed, 54 insertions(+), 63 deletions(-) diff --git a/extractor/cli/go-autobuilder/go-autobuilder.go b/extractor/cli/go-autobuilder/go-autobuilder.go index 0c5bca42d6a..d2306191370 100644 --- a/extractor/cli/go-autobuilder/go-autobuilder.go +++ b/extractor/cli/go-autobuilder/go-autobuilder.go @@ -1,7 +1,6 @@ package main import ( - "errors" "fmt" "golang.org/x/mod/semver" "io/ioutil" @@ -207,51 +206,6 @@ func checkVendor() bool { return true } -func getOsToolsSubdir() (string, error) { - switch runtime.GOOS { - case "darwin": - return "osx64", nil - case "linux": - return "linux64", nil - case "windows": - return "win64", nil - } - return "", errors.New("Unsupported OS: " + runtime.GOOS) -} - -func getExtractorDir() (string, error) { - mypath, err := os.Executable() - if err == nil { - return filepath.Dir(mypath), nil - } - log.Printf("Could not determine path of autobuilder: %v.\n", err) - - // Fall back to rebuilding our own path from the extractor root: - extractorRoot := os.Getenv("CODEQL_EXTRACTOR_GO_ROOT") - if extractorRoot == "" { - return "", errors.New("CODEQL_EXTRACTOR_GO_ROOT not set.\nThis binary should not be run manually; instead, use the CodeQL CLI or VSCode extension. See https://securitylab.github.com/tools/codeql") - } - - osSubdir, err := getOsToolsSubdir() - if err != nil { - return "", err - } - - return filepath.Join(extractorRoot, "tools", osSubdir), nil -} - -func getExtractorPath() (string, error) { - dirname, err := getExtractorDir() - if err != nil { - return "", err - } - extractor := filepath.Join(dirname, "go-extractor") - if runtime.GOOS == "windows" { - extractor = extractor + ".exe" - } - return extractor, nil -} - func main() { if len(os.Args) > 1 { usage() @@ -568,7 +522,7 @@ func main() { } // extract - extractor, err := getExtractorPath() + extractor, err := util.GetExtractorPath() if err != nil { log.Fatalf("Could not determine path of extractor: %v.\n", err) } diff --git a/extractor/extractor.go b/extractor/extractor.go index 29bc0764b67..6995046ff12 100644 --- a/extractor/extractor.go +++ b/extractor/extractor.go @@ -301,7 +301,11 @@ func NewExtraction(buildFlags []string, patterns []string) *Extraction { dbscheme.CompilationsTable.Emit(statWriter, lbl, wd) i = 0 - dbscheme.CompilationArgsTable.Emit(statWriter, lbl, 0, util.GetExtractorPath()) + extractorPath, err := util.GetExtractorPath() + if err != nil { + log.Fatalf("Unable to get extractor path: %s\n", err.Error()) + } + dbscheme.CompilationArgsTable.Emit(statWriter, lbl, 0, extractorPath) i++ for _, flag := range buildFlags { dbscheme.CompilationArgsTable.Emit(statWriter, lbl, i, flag) diff --git a/extractor/util/util.go b/extractor/util/util.go index a5c5227eee6..06ecc704bb0 100644 --- a/extractor/util/util.go +++ b/extractor/util/util.go @@ -1,10 +1,12 @@ package util import ( + "errors" "log" "os" "os/exec" "path/filepath" + "runtime" "strings" ) @@ -141,20 +143,51 @@ func RunCmd(cmd *exec.Cmd) bool { return true } -func GetExtractorPath() string { - if extractorPath != "" { - return extractorPath +func getOsToolsSubdir() (string, error) { + switch runtime.GOOS { + case "darwin": + return "osx64", nil + case "linux": + return "linux64", nil + case "windows": + return "win64", nil } - - root, set := os.LookupEnv("CODEQL_EXTRACTOR_GO_ROOT") - if !set { - log.Fatal("CODEQL_EXTRACTOR_GO_ROOT not set; this binary should be run from the `codeql` CLI.") - } - platform, set := os.LookupEnv("CODEQL_PLATFORM") - if !set { - log.Fatal("CODEQL_PLATFORM not set; this binary should be run from the `codeql` CLI.") - } - - extractorPath = filepath.Join(root, "tools", platform, "go-extractor") - return extractorPath + return "", errors.New("Unsupported OS: " + runtime.GOOS) +} + +func getExtractorDir() (string, error) { + mypath, err := os.Executable() + if err == nil { + return filepath.Dir(mypath), nil + } + log.Printf("Could not determine path of autobuilder: %v.\n", err) + + // Fall back to rebuilding our own path from the extractor root: + extractorRoot := os.Getenv("CODEQL_EXTRACTOR_GO_ROOT") + if extractorRoot == "" { + return "", errors.New("CODEQL_EXTRACTOR_GO_ROOT not set.\nThis binary should not be run manually; instead, use the CodeQL CLI or VSCode extension. See https://securitylab.github.com/tools/codeql") + } + + osSubdir, err := getOsToolsSubdir() + if err != nil { + return "", err + } + + return filepath.Join(extractorRoot, "tools", osSubdir), nil +} + +func GetExtractorPath() (string, error) { + if extractorPath != "" { + return extractorPath, nil + } + + dirname, err := getExtractorDir() + if err != nil { + return "", err + } + extractorPath := filepath.Join(dirname, "go-extractor") + if runtime.GOOS == "windows" { + extractorPath = extractorPath + ".exe" + } + return extractorPath, nil } From 244f66c3582dccbaa4186d2e83f2de96b8485f06 Mon Sep 17 00:00:00 2001 From: Chris Smowton Date: Thu, 25 Mar 2021 14:44:18 +0000 Subject: [PATCH 2/3] Make diagnostics test platform-neutral --- .../extractor-tests/diagnostics/Diagnostics.ql | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ql/test/extractor-tests/diagnostics/Diagnostics.ql b/ql/test/extractor-tests/diagnostics/Diagnostics.ql index a6e3810c39a..dd6bb45e6a0 100644 --- a/ql/test/extractor-tests/diagnostics/Diagnostics.ql +++ b/ql/test/extractor-tests/diagnostics/Diagnostics.ql @@ -43,13 +43,23 @@ class Diagnostic extends @diagnostic { string toString() { result = "error: " + this.getMessage() } } -query predicate qcompilations(Compilation c, File f) { c.compilesFile(_, f) } +/** + * Wraps `Compilation`, removing the `.exe` suffixes from compilation descriptions + * such that this test produces the same results on Windows and non-Windows platforms. + */ +class PlatformNeutralCompilation extends Compilation { + override string toString() { result = super.toString().regexpReplaceAll("\\.exe", "") } +} -query predicate qdiagnostics(Diagnostic d, Compilation c, File f) { +query predicate qcompilations(PlatformNeutralCompilation c, File f) { c.compilesFile(_, f) } + +query predicate qdiagnostics(Diagnostic d, PlatformNeutralCompilation c, File f) { exists(int fileno | d.diagnosticFor(c, fileno, _) | c.compilesFile(fileno, f)) } -query predicate duplicateerrs(Diagnostic d, Diagnostic d1, Compilation c, int fileno, int idx) { +query predicate duplicateerrs( + Diagnostic d, Diagnostic d1, PlatformNeutralCompilation c, int fileno, int idx +) { d != d1 and d.diagnosticFor(c, fileno, idx) and d1.diagnosticFor(c, fileno, idx) From aef0a07a50d7bba5f70c1384e14d63e790395dbe Mon Sep 17 00:00:00 2001 From: Chris Smowton Date: Thu, 25 Mar 2021 16:20:16 +0000 Subject: [PATCH 3/3] Prefer CODEQL_* environment variables when set --- extractor/util/util.go | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/extractor/util/util.go b/extractor/util/util.go index 06ecc704bb0..57f4597ec22 100644 --- a/extractor/util/util.go +++ b/extractor/util/util.go @@ -144,6 +144,13 @@ func RunCmd(cmd *exec.Cmd) bool { } func getOsToolsSubdir() (string, error) { + platform, set := os.LookupEnv("CODEQL_PLATFORM") + if !set { + log.Print("CODEQL_PLATFORM not set; this binary should be run from the `codeql` CLI. Falling back to use `runtime.GOOS`.\n") + } else { + return platform, nil + } + switch runtime.GOOS { case "darwin": return "osx64", nil @@ -156,16 +163,17 @@ func getOsToolsSubdir() (string, error) { } func getExtractorDir() (string, error) { - mypath, err := os.Executable() - if err == nil { - return filepath.Dir(mypath), nil - } - log.Printf("Could not determine path of autobuilder: %v.\n", err) - - // Fall back to rebuilding our own path from the extractor root: extractorRoot := os.Getenv("CODEQL_EXTRACTOR_GO_ROOT") if extractorRoot == "" { - return "", errors.New("CODEQL_EXTRACTOR_GO_ROOT not set.\nThis binary should not be run manually; instead, use the CodeQL CLI or VSCode extension. See https://securitylab.github.com/tools/codeql") + log.Print("CODEQL_EXTRACTOR_GO_ROOT not set.\nThis binary should not be run manually; instead, use the CodeQL CLI or VSCode extension. See https://securitylab.github.com/tools/codeql.\n") + log.Print("Falling back to guess the root based on this executable's path.\n") + + mypath, err := os.Executable() + if err == nil { + return filepath.Dir(mypath), nil + } else { + return "", errors.New("CODEQL_EXTRACTOR_GO_ROOT not set, and could not determine path of this executable: " + err.Error()) + } } osSubdir, err := getOsToolsSubdir()