Merge pull request #517 from smowton/smowton/fix/restore-extraction-under-odasa

Unify two implementations of GetExtractorPath

extractor/cli/go-autobuilder/go-autobuilder.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"errors"
 	"fmt"
 	"golang.org/x/mod/semver"
 	"io/ioutil"
@@ -207,51 +206,6 @@ func checkVendor() bool {
 	return true
 }
 
-func getOsToolsSubdir() (string, error) {
-	switch runtime.GOOS {
-	case "darwin":
-		return "osx64", nil
-	case "linux":
-		return "linux64", nil
-	case "windows":
-		return "win64", nil
-	}
-	return "", errors.New("Unsupported OS: " + runtime.GOOS)
-}
-
-func getExtractorDir() (string, error) {
-	mypath, err := os.Executable()
-	if err == nil {
-		return filepath.Dir(mypath), nil
-	}
-	log.Printf("Could not determine path of autobuilder: %v.\n", err)
-
-	// Fall back to rebuilding our own path from the extractor root:
-	extractorRoot := os.Getenv("CODEQL_EXTRACTOR_GO_ROOT")
-	if extractorRoot == "" {
-		return "", errors.New("CODEQL_EXTRACTOR_GO_ROOT not set.\nThis binary should not be run manually; instead, use the CodeQL CLI or VSCode extension. See https://securitylab.github.com/tools/codeql")
-	}
-
-	osSubdir, err := getOsToolsSubdir()
-	if err != nil {
-		return "", err
-	}
-
-	return filepath.Join(extractorRoot, "tools", osSubdir), nil
-}
-
-func getExtractorPath() (string, error) {
-	dirname, err := getExtractorDir()
-	if err != nil {
-		return "", err
-	}
-	extractor := filepath.Join(dirname, "go-extractor")
-	if runtime.GOOS == "windows" {
-		extractor = extractor + ".exe"
-	}
-	return extractor, nil
-}
-
 func main() {
 	if len(os.Args) > 1 {
 		usage()
@@ -568,7 +522,7 @@ func main() {
 	}
 
 	// extract
-	extractor, err := getExtractorPath()
+	extractor, err := util.GetExtractorPath()
 	if err != nil {
 		log.Fatalf("Could not determine path of extractor: %v.\n", err)
 	}

extractor/extractor.go

@@ -301,7 +301,11 @@ func NewExtraction(buildFlags []string, patterns []string) *Extraction {
 
 	dbscheme.CompilationsTable.Emit(statWriter, lbl, wd)
 	i = 0
-	dbscheme.CompilationArgsTable.Emit(statWriter, lbl, 0, util.GetExtractorPath())
+	extractorPath, err := util.GetExtractorPath()
+	if err != nil {
+		log.Fatalf("Unable to get extractor path: %s\n", err.Error())
+	}
+	dbscheme.CompilationArgsTable.Emit(statWriter, lbl, 0, extractorPath)
 	i++
 	for _, flag := range buildFlags {
 		dbscheme.CompilationArgsTable.Emit(statWriter, lbl, i, flag)

extractor/util/util.go

@@ -1,10 +1,12 @@
 package util
 
 import (
+	"errors"
 	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"strings"
 )
 
@@ -141,20 +143,59 @@ func RunCmd(cmd *exec.Cmd) bool {
 	return true
 }
 
-func GetExtractorPath() string {
-	if extractorPath != "" {
-		return extractorPath
-	}
-
-	root, set := os.LookupEnv("CODEQL_EXTRACTOR_GO_ROOT")
-	if !set {
-		log.Fatal("CODEQL_EXTRACTOR_GO_ROOT not set; this binary should be run from the `codeql` CLI.")
-	}
+func getOsToolsSubdir() (string, error) {
 	platform, set := os.LookupEnv("CODEQL_PLATFORM")
 	if !set {
-		log.Fatal("CODEQL_PLATFORM not set; this binary should be run from the `codeql` CLI.")
+		log.Print("CODEQL_PLATFORM not set; this binary should be run from the `codeql` CLI. Falling back to use `runtime.GOOS`.\n")
+	} else {
+		return platform, nil
 	}
 
-	extractorPath = filepath.Join(root, "tools", platform, "go-extractor")
-	return extractorPath
+	switch runtime.GOOS {
+	case "darwin":
+		return "osx64", nil
+	case "linux":
+		return "linux64", nil
+	case "windows":
+		return "win64", nil
+	}
+	return "", errors.New("Unsupported OS: " + runtime.GOOS)
+}
+
+func getExtractorDir() (string, error) {
+	extractorRoot := os.Getenv("CODEQL_EXTRACTOR_GO_ROOT")
+	if extractorRoot == "" {
+		log.Print("CODEQL_EXTRACTOR_GO_ROOT not set.\nThis binary should not be run manually; instead, use the CodeQL CLI or VSCode extension. See https://securitylab.github.com/tools/codeql.\n")
+		log.Print("Falling back to guess the root based on this executable's path.\n")
+
+		mypath, err := os.Executable()
+		if err == nil {
+			return filepath.Dir(mypath), nil
+		} else {
+			return "", errors.New("CODEQL_EXTRACTOR_GO_ROOT not set, and could not determine path of this executable: " + err.Error())
+		}
+	}
+
+	osSubdir, err := getOsToolsSubdir()
+	if err != nil {
+		return "", err
+	}
+
+	return filepath.Join(extractorRoot, "tools", osSubdir), nil
+}
+
+func GetExtractorPath() (string, error) {
+	if extractorPath != "" {
+		return extractorPath, nil
+	}
+
+	dirname, err := getExtractorDir()
+	if err != nil {
+		return "", err
+	}
+	extractorPath := filepath.Join(dirname, "go-extractor")
+	if runtime.GOOS == "windows" {
+		extractorPath = extractorPath + ".exe"
+	}
+	return extractorPath, nil
 }

ql/test/extractor-tests/diagnostics/Diagnostics.ql

@@ -43,13 +43,23 @@ class Diagnostic extends @diagnostic {
   string toString() { result = "error: " + this.getMessage() }
 }
 
-query predicate qcompilations(Compilation c, File f) { c.compilesFile(_, f) }
+/**
+ * Wraps `Compilation`, removing the `.exe` suffixes from compilation descriptions
+ * such that this test produces the same results on Windows and non-Windows platforms.
+ */
+class PlatformNeutralCompilation extends Compilation {
+  override string toString() { result = super.toString().regexpReplaceAll("\\.exe", "") }
+}
 
-query predicate qdiagnostics(Diagnostic d, Compilation c, File f) {
+query predicate qcompilations(PlatformNeutralCompilation c, File f) { c.compilesFile(_, f) }
+
+query predicate qdiagnostics(Diagnostic d, PlatformNeutralCompilation c, File f) {
   exists(int fileno | d.diagnosticFor(c, fileno, _) | c.compilesFile(fileno, f))
 }
 
-query predicate duplicateerrs(Diagnostic d, Diagnostic d1, Compilation c, int fileno, int idx) {
+query predicate duplicateerrs(
+  Diagnostic d, Diagnostic d1, PlatformNeutralCompilation c, int fileno, int idx
+) {
   d != d1 and
   d.diagnosticFor(c, fileno, idx) and
   d1.diagnosticFor(c, fileno, idx)