Merge branch 'main' into mathiasvp/replace-ast-with-ir-use-usedataflow

cpp/ql/lib/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.5.4
+
+No user-facing changes.
+
 ## 0.5.3
 
 No user-facing changes.

cpp/ql/lib/change-notes/released/0.5.4.md

@@ -0,0 +1,3 @@
+## 0.5.4
+
+No user-facing changes.

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.5.3
+lastReleaseVersion: 0.5.4

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 0.5.4-dev
+version: 0.5.5-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/lib/semmle/code/cpp/Declaration.qll

@@ -619,11 +619,10 @@ private class DirectAccessHolder extends Element {
   /**
    * Like `couldAccessMember` but only contains derivations in which either
    * (5.2), (5.3) or (5.4) must be invoked. In other words, the `this`
-   * parameter is not ignored. This restriction makes it feasible to fully
-   * enumerate this predicate even on large code bases. We check for 11.4 as
-   * part of (5.3), since this further limits the number of tuples produced by
-   * this predicate.
+   * parameter is not ignored. We check for 11.4 as part of (5.3), since
+   * this further limits the number of tuples produced by this predicate.
    */
+  pragma[inline]
   predicate thisCouldAccessMember(Class memberClass, AccessSpecifier memberAccess, Class derived) {
     // Only (5.4) is recursive, and chains of invocations of (5.4) can always
     // be collapsed to one invocation by the transitivity of 11.2/4.
@@ -665,7 +664,9 @@ private class DirectAccessHolder extends Element {
     //    bypasses `p`. Then that path must be public, or we are in case 2.
     exists(AccessSpecifier public | public.hasName("public") |
       exists(Class between, Class p |
-        between.accessOfBaseMember(memberClass, memberAccess).hasName("protected") and
+        between
+            .accessOfBaseMember(pragma[only_bind_into](memberClass), memberAccess)
+            .hasName("protected") and
         this.isFriendOfOrEqualTo(p) and
         (
           // This is case 1 from above. If `p` derives privately from `between`

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll

@@ -456,6 +456,7 @@ module Impl<FullStateConfigSig Config> {
      * The Boolean `cc` records whether the node is reached through an
      * argument in a call.
      */
+    pragma[assume_small_delta]
     private predicate fwdFlow(NodeEx node, Cc cc) {
       sourceNode(node, _) and
       if hasSourceCallCtx() then cc = true else cc = false

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll

@@ -456,6 +456,7 @@ module Impl<FullStateConfigSig Config> {
      * The Boolean `cc` records whether the node is reached through an
      * argument in a call.
      */
+    pragma[assume_small_delta]
     private predicate fwdFlow(NodeEx node, Cc cc) {
       sourceNode(node, _) and
       if hasSourceCallCtx() then cc = true else cc = false

cpp/ql/src/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.5.4
+
+No user-facing changes.
+
 ## 0.5.3
 
 No user-facing changes.

cpp/ql/src/change-notes/released/0.5.4.md

@@ -0,0 +1,3 @@
+## 0.5.4
+
+No user-facing changes.

cpp/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.5.3
+lastReleaseVersion: 0.5.4

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 0.5.4-dev
+version: 0.5.5-dev
 groups: 
   - cpp
   - queries

csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs

@@ -241,7 +241,7 @@ namespace Semmle.Autobuild.Shared
             SourceArchiveDir = RequireEnvironmentVariable(EnvVars.SourceArchiveDir(this.Options.Language));
             DiagnosticsDir = RequireEnvironmentVariable(EnvVars.DiagnosticDir(this.Options.Language));
 
-            this.diagnostics = actions.CreateDiagnosticsWriter(Path.Combine(DiagnosticsDir, $"autobuilder-{DateTime.UtcNow:yyyyMMddHHmm}.jsonc"));
+            this.diagnostics = actions.CreateDiagnosticsWriter(Path.Combine(DiagnosticsDir, $"autobuilder-{DateTime.UtcNow:yyyyMMddHHmm}-{Environment.ProcessId}.jsonc"));
         }
 
         /// <summary>

csharp/autobuilder/Semmle.Autobuild.Shared/BuildActions.cs

@@ -199,12 +199,8 @@ namespace Semmle.Autobuild.Shared
             if (workingDirectory is not null)
                 pi.WorkingDirectory = workingDirectory;
 
-            // Environment variables can only be used when not redirecting stdout
-            if (!redirectStandardOutput)
-            {
-                if (environment is not null)
-                    environment.ForEach(kvp => pi.Environment[kvp.Key] = kvp.Value);
-            }
+            environment?.ForEach(kvp => pi.Environment[kvp.Key] = kvp.Value);
+
             return pi;
         }
 

csharp/ql/campaigns/Solorigate/lib/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.4.4
+
+No user-facing changes.
+
 ## 1.4.3
 
 No user-facing changes.

csharp/ql/campaigns/Solorigate/lib/change-notes/released/1.4.4.md

@@ -0,0 +1,3 @@
+## 1.4.4
+
+No user-facing changes.

csharp/ql/campaigns/Solorigate/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.4.3
+lastReleaseVersion: 1.4.4

csharp/ql/campaigns/Solorigate/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-all
-version: 1.4.4-dev
+version: 1.4.5-dev
 groups:
     - csharp
     - solorigate

csharp/ql/campaigns/Solorigate/src/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.4.4
+
+No user-facing changes.
+
 ## 1.4.3
 
 No user-facing changes.

csharp/ql/campaigns/Solorigate/src/change-notes/released/1.4.4.md

@@ -0,0 +1,3 @@
+## 1.4.4
+
+No user-facing changes.

csharp/ql/campaigns/Solorigate/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.4.3
+lastReleaseVersion: 1.4.4

csharp/ql/campaigns/Solorigate/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-queries
-version: 1.4.4-dev
+version: 1.4.5-dev
 groups:
     - csharp
     - solorigate

csharp/ql/lib/CHANGELOG.md

@@ -1,3 +1,14 @@
+## 0.5.4
+
+### Minor Analysis Improvements
+
+* The query `cs/static-field-written-by-instance` is updated to handle properties.
+* C# 11: Support for explicit interface member implementation of operators.
+* The extraction of member modifiers has been generalized, which could lead to the extraction of more modifiers.
+* C# 11: Added extractor and library support for `file` scoped types.
+* C# 11: Added extractor support for `required` fields and properties.
+* C# 11: Added library support for `checked` operators.
+
 ## 0.5.3
 
 ### Minor Analysis Improvements

csharp/ql/lib/change-notes/2023-02-14-checked-operators.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* C# 11: Added library support for `checked` operators.

csharp/ql/lib/change-notes/2023-02-16-requiredmembers.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* C# 11: Added extractor support for `required` fields and properties.

csharp/ql/lib/change-notes/2023-02-17-filescopedtypes.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* C# 11: Added extractor and library support for `file` scoped types.

csharp/ql/lib/change-notes/2023-02-22-modifierextraction.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The extraction of member modifiers has been generalised, which could lead to the extraction of more modifiers.

csharp/ql/lib/change-notes/2023-02-27-explicitinterfacemember.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* C# 11: Support for explicit interface member implementation of operators.

csharp/ql/lib/change-notes/2023-02-28-staticfieldwrites.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The query `cs/static-field-written-by-instance` is updated to handle properties.

csharp/ql/lib/change-notes/released/0.5.4.md

@@ -0,0 +1,10 @@
+## 0.5.4
+
+### Minor Analysis Improvements
+
+* The query `cs/static-field-written-by-instance` is updated to handle properties.
+* C# 11: Support for explicit interface member implementation of operators.
+* The extraction of member modifiers has been generalized, which could lead to the extraction of more modifiers.
+* C# 11: Added extractor and library support for `file` scoped types.
+* C# 11: Added extractor support for `required` fields and properties.
+* C# 11: Added library support for `checked` operators.

csharp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.5.3
+lastReleaseVersion: 0.5.4

csharp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-all
-version: 0.5.4-dev
+version: 0.5.5-dev
 groups: csharp
 dbscheme: semmlecode.csharp.dbscheme
 extractor: csharp

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll

@@ -456,6 +456,7 @@ module Impl<FullStateConfigSig Config> {
      * The Boolean `cc` records whether the node is reached through an
      * argument in a call.
      */
+    pragma[assume_small_delta]
     private predicate fwdFlow(NodeEx node, Cc cc) {
       sourceNode(node, _) and
       if hasSourceCallCtx() then cc = true else cc = false

csharp/ql/src/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.5.4
+
+No user-facing changes.
+
 ## 0.5.3
 
 No user-facing changes.

csharp/ql/src/change-notes/released/0.5.4.md

@@ -0,0 +1,3 @@
+## 0.5.4
+
+No user-facing changes.

csharp/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.5.3
+lastReleaseVersion: 0.5.4

csharp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-queries
-version: 0.5.4-dev
+version: 0.5.5-dev
 groups: 
   - csharp
   - queries

go/extractor/cli/go-autobuilder/go-autobuilder.go

@@ -16,6 +16,7 @@ import (
 	"golang.org/x/mod/semver"
 
 	"github.com/github/codeql-go/extractor/autobuilder"
+	"github.com/github/codeql-go/extractor/diagnostics"
 	"github.com/github/codeql-go/extractor/util"
 )
 
@@ -203,7 +204,7 @@ func (m ModMode) argsForGoVersion(version string) []string {
 }
 
 // addVersionToMod add a go version directive, e.g. `go 1.14` to a `go.mod` file.
-func addVersionToMod(goMod []byte, version string) bool {
+func addVersionToMod(version string) bool {
 	cmd := exec.Command("go", "mod", "edit", "-go="+version)
 	return util.RunCmd(cmd)
 }
@@ -249,12 +250,29 @@ func main() {
 	depMode := GoGetNoModules
 	modMode := ModUnset
 	needGopath := true
+	goDirectiveFound := false
 	if _, present := os.LookupEnv("GO111MODULE"); !present {
 		os.Setenv("GO111MODULE", "auto")
 	}
 	if util.FileExists("go.mod") {
 		depMode = GoGetWithModules
 		needGopath = false
+		versionRe := regexp.MustCompile(`(?m)^go[ \t\r]+([0-9]+\.[0-9]+)$`)
+		goMod, err := ioutil.ReadFile("go.mod")
+		if err != nil {
+			log.Println("Failed to read go.mod to check for missing Go version")
+		} else {
+			matches := versionRe.FindSubmatch(goMod)
+			if matches != nil {
+				goDirectiveFound = true
+				if len(matches) > 1 {
+					goDirectiveVersion := "v" + string(matches[1])
+					if semver.Compare(goDirectiveVersion, getEnvGoSemVer()) >= 0 {
+						diagnostics.EmitNewerGoVersionNeeded()
+					}
+				}
+			}
+		}
 		log.Println("Found go.mod, enabling go modules")
 	} else if util.FileExists("Gopkg.toml") {
 		depMode = Dep
@@ -283,10 +301,7 @@ func main() {
 		// we work around this by adding an explicit go version of 1.13, which is the last version
 		// where this is not an issue
 		if depMode == GoGetWithModules {
-			goMod, err := ioutil.ReadFile("go.mod")
-			if err != nil {
-				log.Println("Failed to read go.mod to check for missing Go version")
-			} else if versionRe := regexp.MustCompile(`(?m)^go[ \t\r]+[0-9]+\.[0-9]+$`); !versionRe.Match(goMod) {
+			if !goDirectiveFound {
 				// if the go.mod does not contain a version line
 				modulesTxt, err := ioutil.ReadFile("vendor/modules.txt")
 				if err != nil {
@@ -294,7 +309,7 @@ func main() {
 				} else if explicitRe := regexp.MustCompile("(?m)^## explicit$"); !explicitRe.Match(modulesTxt) {
 					// and the modules.txt does not contain an explicit annotation
 					log.Println("Adding a version directive to the go.mod file as the modules.txt does not have explicit annotations")
-					if !addVersionToMod(goMod, "1.13") {
+					if !addVersionToMod("1.13") {
 						log.Println("Failed to add a version to the go.mod file to fix explicitly required package bug; not using vendored dependencies")
 						modMode = ModMod
 					}

go/extractor/diagnostics/diagnostics.go

@@ -0,0 +1,183 @@
+package diagnostics
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+	"time"
+)
+
+type sourceStruct struct {
+	Id            string `json:"id"`
+	Name          string `json:"name"`
+	ExtractorName string `json:"extractorName"`
+}
+
+type diagnosticSeverity string
+
+const (
+	severityError   diagnosticSeverity = "error"
+	severityWarning diagnosticSeverity = "warning"
+	severityNote    diagnosticSeverity = "note"
+)
+
+type visibilityStruct struct {
+	StatusPage      bool `json:"statusPage"`      // True if the message should be displayed on the status page (defaults to false)
+	CliSummaryTable bool `json:"cliSummaryTable"` // True if the message should be counted in the diagnostics summary table printed by `codeql database analyze` (defaults to false)
+	Telemetry       bool `json:"telemetry"`       // True if the message should be sent to telemetry (defaults to false)
+}
+
+var fullVisibility *visibilityStruct = &visibilityStruct{true, true, true}
+
+type locationStruct struct {
+	File        string `json:"file,omitempty"`
+	StartLine   int    `json:"startLine,omitempty"`
+	StartColumn int    `json:"startColumn,omitempty"`
+	EndLine     int    `json:"endLine,omitempty"`
+	EndColumn   int    `json:"endColumn,omitempty"`
+}
+
+var noLocation *locationStruct = nil
+
+type diagnostic struct {
+	Timestamp       string            `json:"timestamp"`
+	Source          sourceStruct      `json:"source"`
+	MarkdownMessage string            `json:"markdownMessage"`
+	Severity        string            `json:"severity"`
+	Visibility      *visibilityStruct `json:"visibility,omitempty"` // Use a pointer so that it is omitted if nil
+	Location        *locationStruct   `json:"location,omitempty"`   // Use a pointer so that it is omitted if nil
+}
+
+var diagnosticsEmitted, diagnosticsLimit uint = 0, 100
+var noDiagnosticDirPrinted bool = false
+
+func emitDiagnostic(sourceid, sourcename, markdownMessage string, severity diagnosticSeverity, visibility *visibilityStruct, location *locationStruct) {
+	if diagnosticsEmitted < diagnosticsLimit {
+		diagnosticsEmitted += 1
+
+		diagnosticDir := os.Getenv("CODEQL_EXTRACTOR_GO_DIAGNOSTIC_DIR")
+		if diagnosticDir == "" {
+			if !noDiagnosticDirPrinted {
+				log.Println("No diagnostic directory set, so not emitting diagnostic")
+				noDiagnosticDirPrinted = true
+			}
+			return
+		}
+
+		timestamp := time.Now().UTC().Format("2006-01-02T15:04:05.000") + "Z"
+
+		var d diagnostic
+
+		if diagnosticsEmitted < diagnosticsLimit {
+			d = diagnostic{
+				timestamp,
+				sourceStruct{sourceid, sourcename, "go"},
+				markdownMessage,
+				string(severity),
+				visibility,
+				location,
+			}
+		} else {
+			d = diagnostic{
+				timestamp,
+				sourceStruct{"go/autobuilder/diagnostic-limit-reached", "Diagnostics limit exceeded", "go"},
+				fmt.Sprintf("CodeQL has produced more than the maximum number of diagnostics. Only the first %d have been reported.", diagnosticsLimit),
+				string(severityWarning),
+				fullVisibility,
+				noLocation,
+			}
+		}
+
+		content, err := json.Marshal(d)
+		if err != nil {
+			log.Println(err)
+			return
+		}
+
+		targetFile, err := os.CreateTemp(diagnosticDir, "go-extractor.*.json")
+		if err != nil {
+			log.Println("Failed to create diagnostic file: ")
+			log.Println(err)
+			return
+		}
+		defer func() {
+			if err := targetFile.Close(); err != nil {
+				log.Println("Failed to close diagnostic file:")
+				log.Println(err)
+			}
+		}()
+
+		_, err = targetFile.Write(content)
+		if err != nil {
+			log.Println("Failed to write to diagnostic file: ")
+			log.Println(err)
+		}
+	}
+}
+
+func EmitPackageDifferentOSArchitecture(pkgPath string) {
+	emitDiagnostic(
+		"go/autobuilder/package-different-os-architecture",
+		"An imported package is intended for a different OS or architecture",
+		"`"+pkgPath+"` could not be imported. Make sure the `GOOS` and `GOARCH` [environment variables are correctly set](https://docs.github.com/en/actions/learn-github-actions/variables#defining-environment-variables-for-a-single-workflow). Alternatively, [change your OS and architecture](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#using-a-github-hosted-runner).",
+		severityWarning,
+		fullVisibility,
+		noLocation,
+	)
+}
+
+const maxNumPkgPaths = 5
+
+func EmitCannotFindPackages(pkgPaths []string) {
+	numPkgPaths := len(pkgPaths)
+
+	ending := "s"
+	if numPkgPaths == 1 {
+		ending = ""
+	}
+
+	numPrinted := numPkgPaths
+	truncated := false
+	if numPrinted > maxNumPkgPaths {
+		numPrinted = maxNumPkgPaths
+		truncated = true
+	}
+
+	secondLine := "`" + strings.Join(pkgPaths[0:numPrinted], "`, `") + "`"
+	if truncated {
+		secondLine += fmt.Sprintf(" and %d more", numPkgPaths-maxNumPkgPaths)
+	}
+
+	emitDiagnostic(
+		"go/autobuilder/package-not-found",
+		"Some packages could not be found",
+		fmt.Sprintf("%d package%s could not be found.\n\n%s.\n\nCheck that the paths are correct and make sure any private packages can be accessed. If any of the packages are present in the repository then you may need a [custom build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).", numPkgPaths, ending, secondLine),
+		severityError,
+		fullVisibility,
+		noLocation,
+	)
+}
+
+func EmitNewerGoVersionNeeded() {
+	emitDiagnostic(
+		"go/autobuilder/newer-go-version-needed",
+		"Newer Go version needed",
+		"The detected version of Go is lower than the version specified in `go.mod`. [Install a newer version](https://github.com/actions/setup-go#basic).",
+		severityError,
+		fullVisibility,
+		noLocation,
+	)
+}
+
+func EmitGoFilesFoundButNotProcessed() {
+	emitDiagnostic(
+		"go/autobuilder/go-files-found-but-not-processed",
+		"Go files were found but not processed",
+		"[Specify a custom build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages) that includes one or more `go build` commands to build the `.go` files to be analyzed.",
+		severityError,
+		fullVisibility,
+		noLocation,
+	)
+}

go/extractor/extractor.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/github/codeql-go/extractor/dbscheme"
+	"github.com/github/codeql-go/extractor/diagnostics"
 	"github.com/github/codeql-go/extractor/srcarchive"
 	"github.com/github/codeql-go/extractor/trap"
 	"github.com/github/codeql-go/extractor/util"
@@ -97,6 +98,13 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 
 	if len(pkgs) == 0 {
 		log.Println("No packages found.")
+
+		wd, err := os.Getwd()
+		if err != nil {
+			log.Printf("Warning: failed to get working directory: %s\n", err.Error())
+		} else if util.FindGoFiles(wd) {
+			diagnostics.EmitGoFilesFoundButNotProcessed()
+		}
 	}
 
 	log.Println("Extracting universe scope.")
@@ -118,6 +126,8 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 		log.Printf("Done running go list deps: resolved %d packages.", len(pkgInfos))
 	}
 
+	pkgsNotFound := make([]string, 0, len(pkgs))
+
 	// Do a post-order traversal and extract the package scope of each package
 	packages.Visit(pkgs, func(pkg *packages.Package) bool {
 		return true
@@ -144,13 +154,27 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 		if len(pkg.Errors) != 0 {
 			log.Printf("Warning: encountered errors extracting package `%s`:", pkg.PkgPath)
 			for i, err := range pkg.Errors {
-				log.Printf("  %s", err.Error())
+				errString := err.Error()
+				log.Printf("  %s", errString)
+
+				if strings.Contains(errString, "build constraints exclude all Go files in ") {
+					// `err` is a NoGoError from the package cmd/go/internal/load, which we cannot access as it is internal
+					diagnostics.EmitPackageDifferentOSArchitecture(pkg.PkgPath)
+				} else if strings.Contains(errString, "cannot find package") ||
+					strings.Contains(errString, "no required module provides package") {
+					pkgsNotFound = append(pkgsNotFound, pkg.PkgPath)
+				}
 				extraction.extractError(tw, err, lbl, i)
 			}
 		}
+
 		log.Printf("Done extracting types for package %s.", pkg.PkgPath)
 	})
 
+	if len(pkgsNotFound) > 0 {
+		diagnostics.EmitCannotFindPackages(pkgsNotFound)
+	}
+
 	for _, pkg := range pkgs {
 		pkgInfo, ok := pkgInfos[pkg.PkgPath]
 		if !ok || pkgInfo.PkgDir == "" {

go/extractor/util/util.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"io"
+	"io/fs"
 	"log"
 	"os"
 	"os/exec"
@@ -281,3 +282,18 @@ func EscapeTrapSpecialChars(s string) string {
 	s = strings.ReplaceAll(s, "#", "#")
 	return s
 }
+
+func FindGoFiles(root string) bool {
+	found := false
+	filepath.WalkDir(root, func(s string, d fs.DirEntry, e error) error {
+		if e != nil {
+			return e
+		}
+		if filepath.Ext(d.Name()) == ".go" {
+			found = true
+			return filepath.SkipAll
+		}
+		return nil
+	})
+	return found
+}

go/ql/integration-tests/all-platforms/go/diagnostics/build-constraints-exclude-all-go-files/diagnostics.expected

@@ -0,0 +1,14 @@
+{
+  "markdownMessage": "`syscall/js` could not be imported. Make sure the `GOOS` and `GOARCH` [environment variables are correctly set](https://docs.github.com/en/actions/learn-github-actions/variables#defining-environment-variables-for-a-single-workflow). Alternatively, [change your OS and architecture](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#using-a-github-hosted-runner).",
+  "severity": "warning",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/package-different-os-architecture",
+    "name": "An imported package is intended for a different OS or architecture"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}

go/ql/integration-tests/all-platforms/go/diagnostics/build-constraints-exclude-all-go-files/test.py

@@ -0,0 +1,9 @@
+import sys
+
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+os.environ['LGTM_INDEX_IMPORT_PATH'] = "test"
+run_codeql_database_create([], lang="go", source="work", db=None)
+
+check_diagnostics()

go/ql/integration-tests/all-platforms/go/diagnostics/build-constraints-exclude-all-go-files/work/go.mod

@@ -0,0 +1,3 @@
+go 1.18
+
+module test

go/ql/integration-tests/all-platforms/go/diagnostics/build-constraints-exclude-all-go-files/work/test.go

@@ -0,0 +1,8 @@
+package test
+
+import "syscall/js"
+
+func Test() {
+	var x js.Error
+	_ = x
+}

go/ql/integration-tests/all-platforms/go/diagnostics/go-files-found-not-processed/diagnostics.expected

@@ -0,0 +1,14 @@
+{
+  "markdownMessage": "[Specify a custom build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages) that includes one or more `go build` commands to build the `.go` files to be analyzed.",
+  "severity": "error",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/go-files-found-but-not-processed",
+    "name": "Go files were found but not processed"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}

go/ql/integration-tests/all-platforms/go/diagnostics/go-files-found-not-processed/test.py

@@ -0,0 +1,9 @@
+import sys
+
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+os.environ['LGTM_INDEX_IMPORT_PATH'] = "test"
+run_codeql_database_create([], lang="go", source="work", db=None, runFunction=runUnsuccessfully)
+
+check_diagnostics()

go/ql/integration-tests/all-platforms/go/diagnostics/go-files-found-not-processed/work/go.mod

@@ -0,0 +1,3 @@
+go 1.18
+
+module test

go/ql/integration-tests/all-platforms/go/diagnostics/go-files-found-not-processed/work/subdir/go.mod

@@ -0,0 +1,3 @@
+go 1.18
+
+module test/subdir

go/ql/integration-tests/all-platforms/go/diagnostics/go-files-found-not-processed/work/subdir/test.go

@@ -0,0 +1,4 @@
+package test/subdir
+
+func Test() {
+}

go/ql/integration-tests/all-platforms/go/diagnostics/newer-go-version-needed/diagnostics.expected

@@ -0,0 +1,14 @@
+{
+  "markdownMessage": "The detected version of Go is lower than the version specified in `go.mod`. [Install a newer version](https://github.com/actions/setup-go#basic).",
+  "severity": "error",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/newer-go-version-needed",
+    "name": "Newer Go version needed"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}

go/ql/integration-tests/all-platforms/go/diagnostics/newer-go-version-needed/test.py

@@ -0,0 +1,9 @@
+import sys
+
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+os.environ['LGTM_INDEX_IMPORT_PATH'] = "test"
+run_codeql_database_create([], lang="go", source="work", db=None)
+
+check_diagnostics()

go/ql/integration-tests/all-platforms/go/diagnostics/newer-go-version-needed/work/go.mod

@@ -0,0 +1,3 @@
+go 999.0
+
+module test

go/ql/integration-tests/all-platforms/go/diagnostics/newer-go-version-needed/work/test.go

@@ -0,0 +1,4 @@
+package test
+
+func Test() {
+}

go/ql/integration-tests/all-platforms/go/diagnostics/no-go-files-found/test.py

@@ -0,0 +1,9 @@
+import sys
+
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+os.environ['LGTM_INDEX_IMPORT_PATH'] = "test"
+run_codeql_database_create([], lang="go", source="work", db=None, runFunction=runUnsuccessfully)
+
+check_diagnostics()

go/ql/integration-tests/all-platforms/go/diagnostics/no-go-files-found/work/test.txt

@@ -0,0 +1 @@
+// The "Go files were found but not processed" diagnostic should not be emitted because there are no go files

go/ql/integration-tests/all-platforms/go/diagnostics/package-not-found-with-go-mod/diagnostics.expected

@@ -0,0 +1,14 @@
+{
+  "markdownMessage": "110 packages could not be found.\n\n`github.com/nosuchorg/nosuchrepo000`, `github.com/nosuchorg/nosuchrepo001`, `github.com/nosuchorg/nosuchrepo002`, `github.com/nosuchorg/nosuchrepo003`, `github.com/nosuchorg/nosuchrepo004` and 105 more.\n\nCheck that the paths are correct and make sure any private packages can be accessed. If any of the packages are present in the repository then you may need a [custom build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+  "severity": "error",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/package-not-found",
+    "name": "Some packages could not be found"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}

go/ql/integration-tests/all-platforms/go/diagnostics/package-not-found-with-go-mod/test.py

@@ -0,0 +1,9 @@
+import sys
+
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+os.environ['LGTM_INDEX_IMPORT_PATH'] = "test"
+run_codeql_database_create([], lang="go", source="work", db=None)
+
+check_diagnostics()

go/ql/integration-tests/all-platforms/go/diagnostics/package-not-found-with-go-mod/work/go.mod

@@ -0,0 +1,7 @@
+go 1.19
+
+require (
+        github.com/linode/linode-docs-theme v0.0.0-20220622135843-166f108e1933
+)
+
+module test

go/ql/integration-tests/all-platforms/go/diagnostics/package-not-found-with-go-mod/work/go.sum

@@ -0,0 +1 @@
+github.com/linode/linode-docs-theme v0.0.0-20220622135843-166f108e1933 h1:QchGQS6xESuyjdlNJEjvq2ftGX0sCTAhPhD5hAOJVMI=

go/ql/integration-tests/all-platforms/go/diagnostics/package-not-found-with-go-mod/work/test.go

@@ -0,0 +1,337 @@
+package test
+
+import (
+	"github.com/nosuchorg/nosuchrepo000"
+	"github.com/nosuchorg/nosuchrepo001"
+	"github.com/nosuchorg/nosuchrepo002"
+	"github.com/nosuchorg/nosuchrepo003"
+	"github.com/nosuchorg/nosuchrepo004"
+	"github.com/nosuchorg/nosuchrepo005"
+	"github.com/nosuchorg/nosuchrepo006"
+	"github.com/nosuchorg/nosuchrepo007"
+	"github.com/nosuchorg/nosuchrepo008"
+	"github.com/nosuchorg/nosuchrepo009"
+	"github.com/nosuchorg/nosuchrepo010"
+	"github.com/nosuchorg/nosuchrepo011"
+	"github.com/nosuchorg/nosuchrepo012"
+	"github.com/nosuchorg/nosuchrepo013"
+	"github.com/nosuchorg/nosuchrepo014"
+	"github.com/nosuchorg/nosuchrepo015"
+	"github.com/nosuchorg/nosuchrepo016"
+	"github.com/nosuchorg/nosuchrepo017"
+	"github.com/nosuchorg/nosuchrepo018"
+	"github.com/nosuchorg/nosuchrepo019"
+	"github.com/nosuchorg/nosuchrepo020"
+	"github.com/nosuchorg/nosuchrepo021"
+	"github.com/nosuchorg/nosuchrepo022"
+	"github.com/nosuchorg/nosuchrepo023"
+	"github.com/nosuchorg/nosuchrepo024"
+	"github.com/nosuchorg/nosuchrepo025"
+	"github.com/nosuchorg/nosuchrepo026"
+	"github.com/nosuchorg/nosuchrepo027"
+	"github.com/nosuchorg/nosuchrepo028"
+	"github.com/nosuchorg/nosuchrepo029"
+	"github.com/nosuchorg/nosuchrepo030"
+	"github.com/nosuchorg/nosuchrepo031"
+	"github.com/nosuchorg/nosuchrepo032"
+	"github.com/nosuchorg/nosuchrepo033"
+	"github.com/nosuchorg/nosuchrepo034"
+	"github.com/nosuchorg/nosuchrepo035"
+	"github.com/nosuchorg/nosuchrepo036"
+	"github.com/nosuchorg/nosuchrepo037"
+	"github.com/nosuchorg/nosuchrepo038"
+	"github.com/nosuchorg/nosuchrepo039"
+	"github.com/nosuchorg/nosuchrepo040"
+	"github.com/nosuchorg/nosuchrepo041"
+	"github.com/nosuchorg/nosuchrepo042"
+	"github.com/nosuchorg/nosuchrepo043"
+	"github.com/nosuchorg/nosuchrepo044"
+	"github.com/nosuchorg/nosuchrepo045"
+	"github.com/nosuchorg/nosuchrepo046"
+	"github.com/nosuchorg/nosuchrepo047"
+	"github.com/nosuchorg/nosuchrepo048"
+	"github.com/nosuchorg/nosuchrepo049"
+	"github.com/nosuchorg/nosuchrepo050"
+	"github.com/nosuchorg/nosuchrepo051"
+	"github.com/nosuchorg/nosuchrepo052"
+	"github.com/nosuchorg/nosuchrepo053"
+	"github.com/nosuchorg/nosuchrepo054"
+	"github.com/nosuchorg/nosuchrepo055"
+	"github.com/nosuchorg/nosuchrepo056"
+	"github.com/nosuchorg/nosuchrepo057"
+	"github.com/nosuchorg/nosuchrepo058"
+	"github.com/nosuchorg/nosuchrepo059"
+	"github.com/nosuchorg/nosuchrepo060"
+	"github.com/nosuchorg/nosuchrepo061"
+	"github.com/nosuchorg/nosuchrepo062"
+	"github.com/nosuchorg/nosuchrepo063"
+	"github.com/nosuchorg/nosuchrepo064"
+	"github.com/nosuchorg/nosuchrepo065"
+	"github.com/nosuchorg/nosuchrepo066"
+	"github.com/nosuchorg/nosuchrepo067"
+	"github.com/nosuchorg/nosuchrepo068"
+	"github.com/nosuchorg/nosuchrepo069"
+	"github.com/nosuchorg/nosuchrepo070"
+	"github.com/nosuchorg/nosuchrepo071"
+	"github.com/nosuchorg/nosuchrepo072"
+	"github.com/nosuchorg/nosuchrepo073"
+	"github.com/nosuchorg/nosuchrepo074"
+	"github.com/nosuchorg/nosuchrepo075"
+	"github.com/nosuchorg/nosuchrepo076"
+	"github.com/nosuchorg/nosuchrepo077"
+	"github.com/nosuchorg/nosuchrepo078"
+	"github.com/nosuchorg/nosuchrepo079"
+	"github.com/nosuchorg/nosuchrepo080"
+	"github.com/nosuchorg/nosuchrepo081"
+	"github.com/nosuchorg/nosuchrepo082"
+	"github.com/nosuchorg/nosuchrepo083"
+	"github.com/nosuchorg/nosuchrepo084"
+	"github.com/nosuchorg/nosuchrepo085"
+	"github.com/nosuchorg/nosuchrepo086"
+	"github.com/nosuchorg/nosuchrepo087"
+	"github.com/nosuchorg/nosuchrepo088"
+	"github.com/nosuchorg/nosuchrepo089"
+	"github.com/nosuchorg/nosuchrepo090"
+	"github.com/nosuchorg/nosuchrepo091"
+	"github.com/nosuchorg/nosuchrepo092"
+	"github.com/nosuchorg/nosuchrepo093"
+	"github.com/nosuchorg/nosuchrepo094"
+	"github.com/nosuchorg/nosuchrepo095"
+	"github.com/nosuchorg/nosuchrepo096"
+	"github.com/nosuchorg/nosuchrepo097"
+	"github.com/nosuchorg/nosuchrepo098"
+	"github.com/nosuchorg/nosuchrepo099"
+	"github.com/nosuchorg/nosuchrepo100"
+	"github.com/nosuchorg/nosuchrepo101"
+	"github.com/nosuchorg/nosuchrepo102"
+	"github.com/nosuchorg/nosuchrepo103"
+	"github.com/nosuchorg/nosuchrepo104"
+	"github.com/nosuchorg/nosuchrepo105"
+	"github.com/nosuchorg/nosuchrepo106"
+	"github.com/nosuchorg/nosuchrepo107"
+	"github.com/nosuchorg/nosuchrepo108"
+	"github.com/nosuchorg/nosuchrepo109"
+)
+
+func Test() {
+	theme000 := nosuchrepo000.Theme{}
+	_ = theme000
+	theme001 := nosuchrepo001.Theme{}
+	_ = theme001
+	theme002 := nosuchrepo002.Theme{}
+	_ = theme002
+	theme003 := nosuchrepo003.Theme{}
+	_ = theme003
+	theme004 := nosuchrepo004.Theme{}
+	_ = theme004
+	theme005 := nosuchrepo005.Theme{}
+	_ = theme005
+	theme006 := nosuchrepo006.Theme{}
+	_ = theme006
+	theme007 := nosuchrepo007.Theme{}
+	_ = theme007
+	theme008 := nosuchrepo008.Theme{}
+	_ = theme008
+	theme009 := nosuchrepo009.Theme{}
+	_ = theme009
+	theme010 := nosuchrepo010.Theme{}
+	_ = theme010
+	theme011 := nosuchrepo011.Theme{}
+	_ = theme011
+	theme012 := nosuchrepo012.Theme{}
+	_ = theme012
+	theme013 := nosuchrepo013.Theme{}
+	_ = theme013
+	theme014 := nosuchrepo014.Theme{}
+	_ = theme014
+	theme015 := nosuchrepo015.Theme{}
+	_ = theme015
+	theme016 := nosuchrepo016.Theme{}
+	_ = theme016
+	theme017 := nosuchrepo017.Theme{}
+	_ = theme017
+	theme018 := nosuchrepo018.Theme{}
+	_ = theme018
+	theme019 := nosuchrepo019.Theme{}
+	_ = theme019
+	theme020 := nosuchrepo020.Theme{}
+	_ = theme020
+	theme021 := nosuchrepo021.Theme{}
+	_ = theme021
+	theme022 := nosuchrepo022.Theme{}
+	_ = theme022
+	theme023 := nosuchrepo023.Theme{}
+	_ = theme023
+	theme024 := nosuchrepo024.Theme{}
+	_ = theme024
+	theme025 := nosuchrepo025.Theme{}
+	_ = theme025
+	theme026 := nosuchrepo026.Theme{}
+	_ = theme026
+	theme027 := nosuchrepo027.Theme{}
+	_ = theme027
+	theme028 := nosuchrepo028.Theme{}
+	_ = theme028
+	theme029 := nosuchrepo029.Theme{}
+	_ = theme029
+	theme030 := nosuchrepo030.Theme{}
+	_ = theme030
+	theme031 := nosuchrepo031.Theme{}
+	_ = theme031
+	theme032 := nosuchrepo032.Theme{}
+	_ = theme032
+	theme033 := nosuchrepo033.Theme{}
+	_ = theme033
+	theme034 := nosuchrepo034.Theme{}
+	_ = theme034
+	theme035 := nosuchrepo035.Theme{}
+	_ = theme035
+	theme036 := nosuchrepo036.Theme{}
+	_ = theme036
+	theme037 := nosuchrepo037.Theme{}
+	_ = theme037
+	theme038 := nosuchrepo038.Theme{}
+	_ = theme038
+	theme039 := nosuchrepo039.Theme{}
+	_ = theme039
+	theme040 := nosuchrepo040.Theme{}
+	_ = theme040
+	theme041 := nosuchrepo041.Theme{}
+	_ = theme041
+	theme042 := nosuchrepo042.Theme{}
+	_ = theme042
+	theme043 := nosuchrepo043.Theme{}
+	_ = theme043
+	theme044 := nosuchrepo044.Theme{}
+	_ = theme044
+	theme045 := nosuchrepo045.Theme{}
+	_ = theme045
+	theme046 := nosuchrepo046.Theme{}
+	_ = theme046
+	theme047 := nosuchrepo047.Theme{}
+	_ = theme047
+	theme048 := nosuchrepo048.Theme{}
+	_ = theme048
+	theme049 := nosuchrepo049.Theme{}
+	_ = theme049
+	theme050 := nosuchrepo050.Theme{}
+	_ = theme050
+	theme051 := nosuchrepo051.Theme{}
+	_ = theme051
+	theme052 := nosuchrepo052.Theme{}
+	_ = theme052
+	theme053 := nosuchrepo053.Theme{}
+	_ = theme053
+	theme054 := nosuchrepo054.Theme{}
+	_ = theme054
+	theme055 := nosuchrepo055.Theme{}
+	_ = theme055
+	theme056 := nosuchrepo056.Theme{}
+	_ = theme056
+	theme057 := nosuchrepo057.Theme{}
+	_ = theme057
+	theme058 := nosuchrepo058.Theme{}
+	_ = theme058
+	theme059 := nosuchrepo059.Theme{}
+	_ = theme059
+	theme060 := nosuchrepo060.Theme{}
+	_ = theme060
+	theme061 := nosuchrepo061.Theme{}
+	_ = theme061
+	theme062 := nosuchrepo062.Theme{}
+	_ = theme062
+	theme063 := nosuchrepo063.Theme{}
+	_ = theme063
+	theme064 := nosuchrepo064.Theme{}
+	_ = theme064
+	theme065 := nosuchrepo065.Theme{}
+	_ = theme065
+	theme066 := nosuchrepo066.Theme{}
+	_ = theme066
+	theme067 := nosuchrepo067.Theme{}
+	_ = theme067
+	theme068 := nosuchrepo068.Theme{}
+	_ = theme068
+	theme069 := nosuchrepo069.Theme{}
+	_ = theme069
+	theme070 := nosuchrepo070.Theme{}
+	_ = theme070
+	theme071 := nosuchrepo071.Theme{}
+	_ = theme071
+	theme072 := nosuchrepo072.Theme{}
+	_ = theme072
+	theme073 := nosuchrepo073.Theme{}
+	_ = theme073
+	theme074 := nosuchrepo074.Theme{}
+	_ = theme074
+	theme075 := nosuchrepo075.Theme{}
+	_ = theme075
+	theme076 := nosuchrepo076.Theme{}
+	_ = theme076
+	theme077 := nosuchrepo077.Theme{}
+	_ = theme077
+	theme078 := nosuchrepo078.Theme{}
+	_ = theme078
+	theme079 := nosuchrepo079.Theme{}
+	_ = theme079
+	theme080 := nosuchrepo080.Theme{}
+	_ = theme080
+	theme081 := nosuchrepo081.Theme{}
+	_ = theme081
+	theme082 := nosuchrepo082.Theme{}
+	_ = theme082
+	theme083 := nosuchrepo083.Theme{}
+	_ = theme083
+	theme084 := nosuchrepo084.Theme{}
+	_ = theme084
+	theme085 := nosuchrepo085.Theme{}
+	_ = theme085
+	theme086 := nosuchrepo086.Theme{}
+	_ = theme086
+	theme087 := nosuchrepo087.Theme{}
+	_ = theme087
+	theme088 := nosuchrepo088.Theme{}
+	_ = theme088
+	theme089 := nosuchrepo089.Theme{}
+	_ = theme089
+	theme090 := nosuchrepo090.Theme{}
+	_ = theme090
+	theme091 := nosuchrepo091.Theme{}
+	_ = theme091
+	theme092 := nosuchrepo092.Theme{}
+	_ = theme092
+	theme093 := nosuchrepo093.Theme{}
+	_ = theme093
+	theme094 := nosuchrepo094.Theme{}
+	_ = theme094
+	theme095 := nosuchrepo095.Theme{}
+	_ = theme095
+	theme096 := nosuchrepo096.Theme{}
+	_ = theme096
+	theme097 := nosuchrepo097.Theme{}
+	_ = theme097
+	theme098 := nosuchrepo098.Theme{}
+	_ = theme098
+	theme099 := nosuchrepo099.Theme{}
+	_ = theme099
+	theme100 := nosuchrepo100.Theme{}
+	_ = theme100
+	theme101 := nosuchrepo101.Theme{}
+	_ = theme101
+	theme102 := nosuchrepo102.Theme{}
+	_ = theme102
+	theme103 := nosuchrepo103.Theme{}
+	_ = theme103
+	theme104 := nosuchrepo104.Theme{}
+	_ = theme104
+	theme105 := nosuchrepo105.Theme{}
+	_ = theme105
+	theme106 := nosuchrepo106.Theme{}
+	_ = theme106
+	theme107 := nosuchrepo107.Theme{}
+	_ = theme107
+	theme108 := nosuchrepo108.Theme{}
+	_ = theme108
+	theme109 := nosuchrepo109.Theme{}
+	_ = theme109
+}

go/ql/integration-tests/all-platforms/go/diagnostics/package-not-found-without-go-mod/diagnostics.expected

@@ -0,0 +1,14 @@
+{
+  "markdownMessage": "1 package could not be found.\n\n`github.com/linode/linode-docs-theme`.\n\nCheck that the paths are correct and make sure any private packages can be accessed. If any of the packages are present in the repository then you may need a [custom build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+  "severity": "error",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/package-not-found",
+    "name": "Some packages could not be found"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}

go/ql/integration-tests/all-platforms/go/diagnostics/package-not-found-without-go-mod/test.py

@@ -0,0 +1,9 @@
+import sys
+
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+os.environ['LGTM_INDEX_IMPORT_PATH'] = "test"
+run_codeql_database_create([], lang="go", source="work", db=None)
+
+check_diagnostics()

go/ql/integration-tests/all-platforms/go/diagnostics/package-not-found-without-go-mod/work/test.go

@@ -0,0 +1,8 @@
+package test
+
+import linodedocstheme "github.com/linode/linode-docs-theme"
+
+func Test() {
+	theme := linodedocstheme.Theme{}
+	_ = theme
+}

go/ql/lib/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.4.4
+
+No user-facing changes.
+
 ## 0.4.3
 
 ### New Features

go/ql/lib/change-notes/released/0.4.4.md

@@ -0,0 +1,3 @@
+## 0.4.4
+
+No user-facing changes.

go/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.4.3
+lastReleaseVersion: 0.4.4

go/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/go-all
-version: 0.4.4-dev
+version: 0.4.5-dev
 groups: go
 dbscheme: go.dbscheme
 extractor: go

go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl.qll

@@ -456,6 +456,7 @@ module Impl<FullStateConfigSig Config> {
      * The Boolean `cc` records whether the node is reached through an
      * argument in a call.
      */
+    pragma[assume_small_delta]
     private predicate fwdFlow(NodeEx node, Cc cc) {
       sourceNode(node, _) and
       if hasSourceCallCtx() then cc = true else cc = false

go/ql/lib/semmle/go/security/IncorrectIntegerConversionLib.qll

@@ -123,7 +123,13 @@ class ConversionWithoutBoundsCheckConfig extends TaintTracking::Configuration {
     )
   }
 
-  override predicate isSink(DataFlow::Node sink) { this.isSinkWithBitSize(sink, sinkBitSize) }
+  override predicate isSink(DataFlow::Node sink) {
+    // We use the argument of the type conversion as the configuration sink so that we
+    // can sanitize the result of the conversion to prevent flow on to further sinks
+    // without needing to use `isSanitizerOut`, which doesn't work with flow states
+    // (and therefore the legacy `TaintTracking::Configuration` class).
+    this.isSinkWithBitSize(sink.getASuccessor(), sinkBitSize)
+  }
 
   override predicate isSanitizer(DataFlow::Node node) {
     // To catch flows that only happen on 32-bit architectures we
@@ -135,10 +141,9 @@ class ConversionWithoutBoundsCheckConfig extends TaintTracking::Configuration {
       g.isBoundFor(bitSize, sinkIsSigned)
     )
     or
-    exists(DataFlow::Node sink, int bitSize |
+    exists(int bitSize |
       isIncorrectIntegerConversion(sourceBitSize, bitSize) and
-      this.isSinkWithBitSize(sink, bitSize) and
-      TaintTracking::localTaintStep(sink, node)
+      this.isSinkWithBitSize(node, bitSize)
     )
   }
 }

go/ql/src/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.4.4
+
+### Minor Analysis Improvements
+
+* The query `go/incorrect-integer-conversion` now correctly recognizes guards of the form `if val <= x` to protect a conversion `uintX(val)` when `x` is in the range `(math.MaxIntX, math.MaxUintX]`.
+
 ## 0.4.3
 
 ### New Queries

go/ql/src/Security/CWE-681/IncorrectIntegerConversionQuery.ql

@@ -19,10 +19,13 @@ import semmle.go.security.IncorrectIntegerConversionLib
 
 from
   DataFlow::PathNode source, DataFlow::PathNode sink, ConversionWithoutBoundsCheckConfig cfg,
-  DataFlow::CallNode call
-where cfg.hasFlowPath(source, sink) and call.getResult(0) = source.getNode()
-select sink.getNode(), source, sink,
+  DataFlow::CallNode call, DataFlow::Node sinkConverted
+where
+  cfg.hasFlowPath(source, sink) and
+  call.getResult(0) = source.getNode() and
+  sinkConverted = sink.getNode().getASuccessor()
+select sinkConverted, source, sink,
   "Incorrect conversion of " +
     describeBitSize(cfg.getSourceBitSize(), getIntTypeBitSize(source.getNode().getFile())) +
-    " from $@ to a lower bit size type " + sink.getNode().getType().getUnderlyingType().getName() +
+    " from $@ to a lower bit size type " + sinkConverted.getType().getUnderlyingType().getName() +
     " without an upper bound check.", source, call.getTarget().getQualifiedName()

go/ql/src/change-notes/released/0.4.4.md

@@ -1,4 +1,5 @@
----
-category: minorAnalysis
----
-* The query `go/incorrect-integer-conversion` now correctly recognises guards of the form `if val <= x` to protect a conversion `uintX(val)` when `x` is in the range `(math.MaxIntX, math.MaxUintX]`.
+## 0.4.4
+
+### Minor Analysis Improvements
+
+* The query `go/incorrect-integer-conversion` now correctly recognizes guards of the form `if val <= x` to protect a conversion `uintX(val)` when `x` is in the range `(math.MaxIntX, math.MaxUintX]`.

go/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.4.3
+lastReleaseVersion: 0.4.4

go/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/go-queries
-version: 0.4.4-dev
+version: 0.4.5-dev
 groups:
   - go
   - queries

go/ql/test/query-tests/Security/CWE-681/IncorrectIntegerConversion.ql

@@ -1,11 +1,23 @@
 import go
-import TestUtilities.InlineFlowTest
+import TestUtilities.InlineExpectationsTest
 import semmle.go.security.IncorrectIntegerConversionLib
 
-class IncorrectIntegerConversionTest extends InlineFlowTest {
-  override DataFlow::Configuration getValueFlowConfig() {
-    result = any(ConversionWithoutBoundsCheckConfig config)
-  }
+class TestIncorrectIntegerConversion extends InlineExpectationsTest {
+  TestIncorrectIntegerConversion() { this = "TestIncorrectIntegerConversion" }
 
-  override DataFlow::Configuration getTaintFlowConfig() { none() }
+  override string getARelevantTag() { result = "hasValueFlow" }
+
+  override predicate hasActualResult(Location location, string element, string tag, string value) {
+    tag = "hasValueFlow" and
+    exists(DataFlow::Node sink, DataFlow::Node sinkConverted |
+      any(ConversionWithoutBoundsCheckConfig config).hasFlowTo(sink) and
+      sinkConverted = sink.getASuccessor()
+    |
+      sinkConverted
+          .hasLocationInfo(location.getFile().getAbsolutePath(), location.getStartLine(),
+            location.getStartColumn(), location.getEndLine(), location.getEndColumn()) and
+      element = sinkConverted.toString() and
+      value = "\"" + sinkConverted.toString() + "\""
+    )
+  }
 }

java/ql/lib/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.5.4
+
+### Minor Analysis Improvements
+
+* Added new sinks for `java/hardcoded-credential-api-call` to identify the use of hardcoded secrets in the creation and verification of JWT tokens using `com.auth0.jwt`. These sinks are from [an experimental query submitted by @luchua](https://github.com/github/codeql/pull/9036).
+* The Java extractor now supports builds against JDK 20.
+* The query `java/hardcoded-credential-api-call` now recognizes methods that accept user and password from the SQLServerDataSource class of the Microsoft JDBC Driver for SQL Server.
+
 ## 0.5.3
 
 ### New Features

java/ql/lib/change-notes/2023-02-15-mssql-jdbc-hardcoded-credential-api.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The query `java/hardcoded-credential-api-call` now recognizes methods that accept user and password from the SQLServerDataSource class of the Microsoft JDBC Driver for SQL Server.

java/ql/lib/change-notes/2023-02-17-add-hardcoded-secret-for-jwt-tokens.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* Added new sinks for `java/hardcoded-credential-api-call` to identify the use of hardcoded secrets in the creation and verification of JWT tokens using `com.auth0.jwt`. These sinks are from [an experimental query submitted by @luchua](https://github.com/github/codeql/pull/9036).

java/ql/lib/change-notes/2023-02-17-jdk20.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The Java extractor now supports builds against JDK 20.

java/ql/lib/change-notes/released/0.5.4.md

@@ -0,0 +1,7 @@
+## 0.5.4
+
+### Minor Analysis Improvements
+
+* Added new sinks for `java/hardcoded-credential-api-call` to identify the use of hardcoded secrets in the creation and verification of JWT tokens using `com.auth0.jwt`. These sinks are from [an experimental query submitted by @luchua](https://github.com/github/codeql/pull/9036).
+* The Java extractor now supports builds against JDK 20.
+* The query `java/hardcoded-credential-api-call` now recognizes methods that accept user and password from the SQLServerDataSource class of the Microsoft JDBC Driver for SQL Server.

java/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.5.3
+lastReleaseVersion: 0.5.4

java/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-all
-version: 0.5.4-dev
+version: 0.5.5-dev
 groups: java
 dbscheme: config/semmlecode.dbscheme
 extractor: java

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll

@@ -456,6 +456,7 @@ module Impl<FullStateConfigSig Config> {
      * The Boolean `cc` records whether the node is reached through an
      * argument in a call.
      */
+    pragma[assume_small_delta]
     private predicate fwdFlow(NodeEx node, Cc cc) {
       sourceNode(node, _) and
       if hasSourceCallCtx() then cc = true else cc = false

java/ql/src/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.5.4
+
+No user-facing changes.
+
 ## 0.5.3
 
 ### New Queries

java/ql/src/change-notes/released/0.5.4.md

@@ -0,0 +1,3 @@
+## 0.5.4
+
+No user-facing changes.

java/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.5.3
+lastReleaseVersion: 0.5.4

java/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-queries
-version: 0.5.4-dev
+version: 0.5.5-dev
 groups: 
   - java
   - queries

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -1,22 +1,26 @@
 package com.semmle.js.extractor;
 
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.Reader;
 import java.lang.ProcessBuilder.Redirect;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryNotEmptyException;
 import java.nio.file.FileVisitResult;
 import java.nio.file.FileVisitor;
 import java.nio.file.Files;
 import java.nio.file.InvalidPathException;
+import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
@@ -27,6 +31,7 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -41,11 +46,16 @@ import com.semmle.js.extractor.FileExtractor.FileType;
 import com.semmle.js.extractor.trapcache.DefaultTrapCache;
 import com.semmle.js.extractor.trapcache.DummyTrapCache;
 import com.semmle.js.extractor.trapcache.ITrapCache;
+import com.semmle.js.parser.ParseError;
 import com.semmle.js.parser.ParsedProject;
 import com.semmle.ts.extractor.TypeExtractor;
 import com.semmle.ts.extractor.TypeScriptParser;
+import com.semmle.ts.extractor.TypeScriptWrapperOOMError;
 import com.semmle.ts.extractor.TypeTable;
 import com.semmle.util.data.StringUtil;
+import com.semmle.util.diagnostics.DiagnosticLevel;
+import com.semmle.util.diagnostics.DiagnosticWriter;
+import com.semmle.util.diagnostics.DiagnosticLocation;
 import com.semmle.util.exception.CatastrophicError;
 import com.semmle.util.exception.Exceptions;
 import com.semmle.util.exception.ResourceError;
@@ -444,35 +454,143 @@ public class AutoBuild {
 
   /** Perform extraction. */
   public int run() throws IOException {
-    startThreadPool();
-    try {
-      CompletableFuture<?> sourceFuture = extractSource();
-      sourceFuture.join(); // wait for source extraction to complete
-      if (hasSeenCode()) { // don't bother with the externs if no code was seen
-        extractExterns();
+      startThreadPool();
+      try {
+        CompletableFuture<?> sourceFuture = extractSource();
+        sourceFuture.join(); // wait for source extraction to complete
+        if (hasSeenCode()) { // don't bother with the externs if no code was seen
+          extractExterns();
+        }
+        extractXml();
+      } catch (OutOfMemoryError oom) {
+        System.err.println("Out of memory while extracting the project.");
+        return 137; // the CodeQL CLI will interpret this as an out-of-memory error
+        // purpusely not doing anything else (printing stack, etc.), as the JVM
+        // basically guarantees nothing after an OOM
+      } catch (TypeScriptWrapperOOMError oom) {
+        System.err.println("Out of memory while extracting the project.");
+        System.err.println(oom.getMessage());
+        oom.printStackTrace(System.err);
+        return 137;
+      } catch (RuntimeException | IOException e) {
+        writeDiagnostics("Internal error: " + e, JSDiagnosticKind.INTERNAL_ERROR);
+        e.printStackTrace(System.err);
+        return 1;
+      } finally {
+        shutdownThreadPool();
+        diagnosticsToClose.forEach(DiagnosticWriter::close);
       }
-      extractXml();
-    } finally {
-      shutdownThreadPool();
-    }
-    if (!hasSeenCode()) {
-      if (seenFiles) {
-        warn("Only found JavaScript or TypeScript files that were empty or contained syntax errors.");
-      } else {
-        warn("No JavaScript or TypeScript code found.");
+
+      if (!hasSeenCode()) {
+        if (seenFiles) {
+          warn("Only found JavaScript or TypeScript files that were empty or contained syntax errors.");
+        } else {
+          warn("No JavaScript or TypeScript code found.");
+        }
+        // ensuring that the finalize steps detects that no code was seen.
+        Path srcFolder = Paths.get(EnvironmentVariables.getWipDatabase(), "src");
+        try {
+          // Non-recursive delete because "src/" should be empty.
+          FileUtil8.delete(srcFolder);
+        } catch (NoSuchFileException e) {
+          Exceptions.ignore(e, "the directory did not exist");
+        } catch (DirectoryNotEmptyException e) {
+          Exceptions.ignore(e, "just leave the directory if it is not empty");
+        }
+        return 0;
       }
-      // ensuring that the finalize steps detects that no code was seen. 
-      Path srcFolder = Paths.get(EnvironmentVariables.getWipDatabase(), "src");
-      // check that the srcFolder is empty
-      if (Files.list(srcFolder).count() == 0) {
-        // Non-recursive delete because "src/" should be empty.
-        FileUtil8.delete(srcFolder);
-      }
-      return 0;
-    }
     return 0;
   }
 
+  /**
+   * A kind of error that can happen during extraction of JavaScript or TypeScript
+   * code.
+   * For use with the {@link #writeDiagnostics(String, JSDiagnosticKind)} method.
+   */
+  public static enum JSDiagnosticKind {
+    PARSE_ERROR("parse-error", "Parse error", DiagnosticLevel.Warning),
+    INTERNAL_ERROR("internal-error", "Internal error", DiagnosticLevel.Debug);
+
+    private final String id;
+    private final String name;
+    private final DiagnosticLevel level;
+
+    private JSDiagnosticKind(String id, String name, DiagnosticLevel level) {
+      this.id = id;
+      this.name = name;
+      this.level = level;
+    }
+
+    public String getId() {
+      return id;
+    }
+
+    public String getName() {
+      return name;
+    }
+
+    public DiagnosticLevel getLevel() {
+      return level;
+    }
+  }
+
+  private AtomicInteger diagnosticCount = new AtomicInteger(0);
+  private List<DiagnosticWriter> diagnosticsToClose = Collections.synchronizedList(new ArrayList<>());
+  private ThreadLocal<DiagnosticWriter> diagnostics = new ThreadLocal<DiagnosticWriter>(){
+        @Override protected DiagnosticWriter initialValue() {
+            DiagnosticWriter result = initDiagnosticsWriter(diagnosticCount.incrementAndGet());
+            diagnosticsToClose.add(result);
+            return result;
+        }
+  };
+
+  /**
+   * Persist a diagnostic message to a file in the diagnostics directory.
+   * See {@link JSDiagnosticKind} for the kinds of errors that can be reported,
+   * and see
+   * {@link DiagnosticWriter} for more details.
+   */
+  public void writeDiagnostics(String message, JSDiagnosticKind error) throws IOException {
+    writeDiagnostics(message, error, null);
+  }
+
+
+  /**
+   * Persist a diagnostic message with a location to a file in the diagnostics directory.
+   * See {@link JSDiagnosticKind} for the kinds of errors that can be reported,
+   * and see
+   * {@link DiagnosticWriter} for more details.
+   */
+  public void writeDiagnostics(String message, JSDiagnosticKind error, DiagnosticLocation location) throws IOException {
+    if (diagnostics.get() == null) {
+      warn("No diagnostics directory, so not writing diagnostic: " + message);
+      return;
+    }
+
+    // DiagnosticLevel level, String extractorName, String sourceId, String sourceName, String markdown
+    diagnostics.get().writeMarkdown(error.getLevel(), "javascript", "javascript/" + error.getId(), error.getName(),
+        message, location);
+  }
+
+  private DiagnosticWriter initDiagnosticsWriter(int count) {
+    String diagnosticsDir = System.getenv("CODEQL_EXTRACTOR_JAVASCRIPT_DIAGNOSTIC_DIR");
+
+    if (diagnosticsDir != null) {
+      File diagnosticsDirFile = new File(diagnosticsDir);
+      if (!diagnosticsDirFile.isDirectory()) {
+        warn("Diagnostics directory " + diagnosticsDir + " does not exist");
+      } else {
+        File diagnosticsFile = new File(diagnosticsDirFile, "autobuilder-" + count + ".jsonl");
+        try {
+          return new DiagnosticWriter(diagnosticsFile);
+        } catch (FileNotFoundException e) {
+          warn("Failed to open diagnostics file " + diagnosticsFile);
+        }
+      }
+    }
+    return null;
+  }
+
   private void startThreadPool() {
     int defaultNumThreads = 1;
     int numThreads = Env.systemEnv().getInt("LGTM_THREADS", defaultNumThreads);
@@ -1113,13 +1231,38 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
 
     try {
       long start = logBeginProcess("Extracting " + file);
-      Integer loc = extractor.extract(f, state);
-      if (!extractor.getConfig().isExterns() && (loc == null || loc != 0)) seenCode = true;
+      ParseResultInfo loc = extractor.extract(f, state);
+      if (!extractor.getConfig().isExterns() && (loc == null || loc.getLinesOfCode() != 0)) seenCode = true;
       if (!extractor.getConfig().isExterns()) seenFiles = true;
+      for (ParseError err : loc.getParseErrors()) {
+        String msg = "A parse error occurred: " + StringUtil.escapeMarkdown(err.getMessage())
+            + ". Check the syntax of the file. If the file is invalid, correct the error or [exclude](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/customizing-code-scanning) the file from analysis.";
+        // file, relative to the source root
+        String relativeFilePath = null;
+        if (file.startsWith(LGTM_SRC)) {
+          relativeFilePath = file.subpath(LGTM_SRC.getNameCount(), file.getNameCount()).toString();
+        }
+        DiagnosticLocation diagLoc = DiagnosticLocation.builder()
+            .setFile(relativeFilePath)
+            .setStartLine(err.getPosition().getLine())
+            .setStartColumn(err.getPosition().getColumn())
+            .setEndLine(err.getPosition().getLine())
+            .setEndColumn(err.getPosition().getColumn())
+            .build();
+        writeDiagnostics(msg, JSDiagnosticKind.PARSE_ERROR, diagLoc);
+      }
       logEndProcess(start, "Done extracting " + file);
+    } catch (OutOfMemoryError oom) {
+      System.err.println("Out of memory while extracting a file.");
+      System.exit(137); // caught by the CodeQL CLI
     } catch (Throwable t) {
       System.err.println("Exception while extracting " + file + ".");
       t.printStackTrace(System.err);
+      try {
+        writeDiagnostics("Internal error: " + t, JSDiagnosticKind.INTERNAL_ERROR);
+      } catch (IOException ignored) {
+        Exceptions.ignore(ignored, "we are already crashing");
+      }
       System.exit(1);
     }
   }

javascript/extractor/src/com/semmle/js/extractor/FileExtractor.java

@@ -5,7 +5,6 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileReader;
 import java.io.IOException;
-import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.util.LinkedHashSet;
@@ -434,7 +433,7 @@ public class FileExtractor {
   }
 
   /** @return the number of lines of code extracted, or {@code null} if the file was cached */
-  public Integer extract(File f, ExtractorState state) throws IOException {
+  public ParseResultInfo extract(File f, ExtractorState state) throws IOException {
     FileSnippet snippet = state.getSnippets().get(f.toPath());
     if (snippet != null) {
       return this.extractSnippet(f.toPath(), snippet, state);
@@ -461,7 +460,7 @@ public class FileExtractor {
    * <p>A trap file will be derived from the snippet file, but its file label, source locations, and
    * source archive entry are based on the original file.
    */
-  private Integer extractSnippet(Path file, FileSnippet origin, ExtractorState state) throws IOException {
+  private ParseResultInfo extractSnippet(Path file, FileSnippet origin, ExtractorState state) throws IOException {
     TrapWriter trapwriter = outputConfig.getTrapWriterFactory().mkTrapWriter(file.toFile());
 
     File originalFile = origin.getOriginalFile().toFile();
@@ -495,7 +494,7 @@ public class FileExtractor {
    * <p>Also note that we support extraction with TRAP writer factories that are not file-backed;
    * obviously, no caching is done in that scenario.
    */
-  private Integer extractContents(
+  private ParseResultInfo extractContents(
       File extractedFile, Label fileLabel, String source, LocationManager locationManager, ExtractorState state)
       throws IOException {
     ExtractionMetrics metrics = new ExtractionMetrics();
@@ -545,7 +544,7 @@ public class FileExtractor {
       TextualExtractor textualExtractor =
           new TextualExtractor(
               trapwriter, locationManager, source, config.getExtractLines(), metrics, extractedFile);
-      LoCInfo loc = extractor.extract(textualExtractor);
+      ParseResultInfo loc = extractor.extract(textualExtractor);
       int numLines = textualExtractor.isSnippet() ? 0 : textualExtractor.getNumLines();
       int linesOfCode = loc.getLinesOfCode(), linesOfComments = loc.getLinesOfComments();
       trapwriter.addTuple("numlines", fileLabel, numLines, linesOfCode, linesOfComments);
@@ -553,7 +552,7 @@ public class FileExtractor {
       metrics.stopPhase(ExtractionPhase.FileExtractor_extractContents);
       metrics.writeTimingsToTrap(trapwriter);
       successful = true;
-      return linesOfCode;
+      return loc;
     } finally {
       if (!successful && trapwriter instanceof CachingTrapWriter)
         ((CachingTrapWriter) trapwriter).discard();

javascript/extractor/src/com/semmle/js/extractor/HTMLExtractor.java

@@ -3,6 +3,7 @@ package com.semmle.js.extractor;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.Collections;
 import java.util.List;
 import java.util.function.Supplier;
 import java.util.regex.Matcher;
@@ -29,7 +30,7 @@ import net.htmlparser.jericho.Source;
 
 /** Extractor for handling HTML and XHTML files. */
 public class HTMLExtractor implements IExtractor {
-  private LoCInfo locInfo = new LoCInfo(0, 0);
+  private ParseResultInfo locInfo = new ParseResultInfo(0, 0, Collections.emptyList());
 
   private class JavaScriptHTMLElementHandler implements HtmlPopulator.ElementHandler {
     private final ScopeManager scopeManager;
@@ -212,11 +213,11 @@ public class HTMLExtractor implements IExtractor {
   }
 
   @Override
-  public LoCInfo extract(TextualExtractor textualExtractor) throws IOException {
+  public ParseResultInfo extract(TextualExtractor textualExtractor) throws IOException {
     return extractEx(textualExtractor).snd();
   }
 
-  public Pair<List<Label>, LoCInfo> extractEx(TextualExtractor textualExtractor) {
+  public Pair<List<Label>, ParseResultInfo> extractEx(TextualExtractor textualExtractor) {
     // Angular templates contain attribute names that are not valid HTML/XML, such
     // as [foo], (foo), [(foo)], and *foo.
     // Allow a large number of errors in attribute names, so the Jericho parser does
@@ -369,7 +370,7 @@ public class HTMLExtractor implements IExtractor {
               config.getExtractLines(),
               textualExtractor.getMetrics(),
               textualExtractor.getExtractedFile());
-      Pair<Label, LoCInfo> result = extractor.extract(tx, source, toplevelKind, scopeManager);
+      Pair<Label, ParseResultInfo> result = extractor.extract(tx, source, toplevelKind, scopeManager);
       Label toplevelLabel = result.fst();
       if (toplevelLabel != null) { // can be null when script ends up being parsed as JSON
         emitTopLevelXmlNodeBinding(parentLabel, toplevelLabel, trapWriter);

javascript/extractor/src/com/semmle/js/extractor/IExtractor.java

@@ -1,6 +1,7 @@
 package com.semmle.js.extractor;
 
 import java.io.IOException;
+import com.semmle.js.parser.ParseError;
 
 /** Generic extractor interface. */
 public interface IExtractor {
@@ -9,5 +10,5 @@ public interface IExtractor {
    * TextualExtractor}, and return information about the number of lines of code and the number of
    * lines of comments extracted.
    */
-  public LoCInfo extract(TextualExtractor textualExtractor) throws IOException;
+  public ParseResultInfo extract(TextualExtractor textualExtractor) throws IOException;
 }

javascript/extractor/src/com/semmle/js/extractor/JSExtractor.java

@@ -1,6 +1,7 @@
 package com.semmle.js.extractor;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -18,6 +19,7 @@ import com.semmle.util.exception.Exceptions;
 import com.semmle.util.exception.UserError;
 import com.semmle.util.trap.TrapWriter;
 import com.semmle.util.trap.TrapWriter.Label;
+import com.semmle.js.extractor.ParseResultInfo;
 
 /**
  * Extractor for populating JavaScript source code, including AST information, lexical information
@@ -36,14 +38,14 @@ public class JSExtractor {
   private static final Pattern containsModuleIndicator =
       Pattern.compile("(?m)^([ \t]*)(import|export|goog\\.module)\\b");
 
-  public Pair<Label, LoCInfo> extract(
+  public Pair<Label, ParseResultInfo> extract(
       TextualExtractor textualExtractor, String source, TopLevelKind toplevelKind, ScopeManager scopeManager)
       throws ParseError {
     // if the file starts with `{ "<string>":` it won't parse as JavaScript; try parsing as JSON
     // instead
     if (FileExtractor.JSON_OBJECT_START.matcher(textualExtractor.getSource()).matches()) {
       try {
-        LoCInfo loc =
+        ParseResultInfo loc =
             new JSONExtractor(config.withTolerateParseErrors(false)).extract(textualExtractor);
         return Pair.make(null, loc);
       } catch (UserError ue) {
@@ -82,7 +84,7 @@ public class JSExtractor {
     return SourceType.SCRIPT;
   }
 
-  public Pair<Label, LoCInfo> extract(
+  public Pair<Label, ParseResultInfo> extract(
       TextualExtractor textualExtractor,
       String source,
       TopLevelKind toplevelKind,
@@ -97,7 +99,7 @@ public class JSExtractor {
     Platform platform = config.getPlatform();
     Node ast = parserRes.getAST();
     LexicalExtractor lexicalExtractor;
-    LoCInfo loc;
+    ParseResultInfo loc;
     if (ast != null) {
       platform = getPlatform(platform, ast);
       if (sourceType == SourceType.SCRIPT && platform == Platform.NODE) {
@@ -124,9 +126,10 @@ public class JSExtractor {
 
       trapwriter.addTuple("toplevels", toplevelLabel, toplevelKind.getValue());
       locationManager.emitSnippetLocation(toplevelLabel, 1, 1, 1, 1);
-      loc = new LoCInfo(0, 0);
+      loc = new ParseResultInfo(0, 0, Collections.emptyList());
     }
 
+    loc.addParseErrors(parserRes.getErrors());
     for (ParseError parseError : parserRes.getErrors()) {
       if (!config.isTolerateParseErrors()) throw parseError;
       Label key = trapwriter.freshLabel();

javascript/extractor/src/com/semmle/js/extractor/JSONExtractor.java

@@ -10,6 +10,8 @@ import com.semmle.js.parser.ParseError;
 import com.semmle.util.data.Pair;
 import com.semmle.util.trap.TrapWriter;
 import com.semmle.util.trap.TrapWriter.Label;
+import com.semmle.js.extractor.ParseResultInfo;
+import java.util.Collections;
 import java.util.List;
 
 /** Extractor for populating JSON files. */
@@ -31,12 +33,12 @@ public class JSONExtractor implements IExtractor {
   }
 
   @Override
-  public LoCInfo extract(final TextualExtractor textualExtractor) {
+  public ParseResultInfo extract(final TextualExtractor textualExtractor) {
     final TrapWriter trapwriter = textualExtractor.getTrapwriter();
     final LocationManager locationManager = textualExtractor.getLocationManager();
     try {
       String source = textualExtractor.getSource();
-      Pair<JSONValue, List<ParseError>> res = new JSONParser().parseValue(source);
+      Pair<JSONValue, List<ParseError>> res = JSONParser.parseValue(source);
       JSONValue v = res.fst();
       List<ParseError> recoverableErrors = res.snd();
       if (!recoverableErrors.isEmpty() && !tolerateParseErrors)
@@ -90,13 +92,14 @@ public class JSONExtractor implements IExtractor {
 
       for (ParseError e : recoverableErrors)
         populateError(textualExtractor, trapwriter, locationManager, e);
+
+      return new ParseResultInfo(0, 0, recoverableErrors);
     } catch (ParseError e) {
       if (!this.tolerateParseErrors) throw e.asUserError();
 
       populateError(textualExtractor, trapwriter, locationManager, e);
+      return new ParseResultInfo(0, 0, Collections.emptyList());
     }
-
-    return new LoCInfo(0, 0);
   }
 
   private void populateError(

javascript/extractor/src/com/semmle/js/extractor/LexicalExtractor.java

@@ -1,5 +1,6 @@
 package com.semmle.js.extractor;
 
+import java.util.Collections;
 import java.util.List;
 
 import com.semmle.js.ast.Comment;
@@ -50,10 +51,10 @@ public class LexicalExtractor {
     return textualExtractor.getMetrics();
   }
 
-  public LoCInfo extractLines(String src, Label toplevelKey) {
+  public ParseResultInfo extractLines(String src, Label toplevelKey) {
     textualExtractor.getMetrics().startPhase(ExtractionPhase.LexicalExtractor_extractLines);
     Position end = textualExtractor.extractLines(src, toplevelKey);
-    LoCInfo info = emitNumlines(toplevelKey, new Position(1, 0, 0), end);
+    ParseResultInfo info = emitNumlines(toplevelKey, new Position(1, 0, 0), end);
     textualExtractor.getMetrics().stopPhase(ExtractionPhase.LexicalExtractor_extractLines);
     return info;
   }
@@ -65,7 +66,7 @@ public class LexicalExtractor {
    * @param start the start position of the node
    * @param end the end position of the node
    */
-  public LoCInfo emitNumlines(Label key, Position start, Position end) {
+  public ParseResultInfo emitNumlines(Label key, Position start, Position end) {
     int num_code = 0, num_comment = 0, num_lines = end.getLine() - start.getLine() + 1;
 
     if (tokens != null && comments != null) {
@@ -104,7 +105,7 @@ public class LexicalExtractor {
     }
 
     trapwriter.addTuple("numlines", key, num_lines, num_code, num_comment);
-    return new LoCInfo(num_code, num_comment);
+    return new ParseResultInfo(num_code, num_comment, Collections.emptyList());
   }
 
   private <T extends SourceElement> int findNode(List<T> ts, Position start) {

javascript/extractor/src/com/semmle/js/extractor/LoCInfo.java

@@ -1,27 +0,0 @@
-package com.semmle.js.extractor;
-
-/**
- * Utility class for representing LoC information; really just a glorified <code>
- * Pair&lt;Integer, Integer&gt;</code>.
- */
-public class LoCInfo {
-  private int linesOfCode, linesOfComments;
-
-  public LoCInfo(int linesOfCode, int linesOfComments) {
-    this.linesOfCode = linesOfCode;
-    this.linesOfComments = linesOfComments;
-  }
-
-  public void add(LoCInfo that) {
-    this.linesOfCode += that.linesOfCode;
-    this.linesOfComments += that.linesOfComments;
-  }
-
-  public int getLinesOfCode() {
-    return linesOfCode;
-  }
-
-  public int getLinesOfComments() {
-    return linesOfComments;
-  }
-}