Merge pull request #15833 from github/release-prep/2.16.4

Release preparation for version 2.16.4

cpp/ql/lib/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.12.7
+
+### Minor Analysis Improvements
+
+* Added destructors for named objects to the intermediate representation.
+
 ## 0.12.6
 
 ### New Features

cpp/ql/lib/change-notes/released/0.12.7.md

@@ -1,4 +1,5 @@
----
-category: minorAnalysis
----
-* Added destructors for named objects to the intermediate representation.
+## 0.12.7
+
+### Minor Analysis Improvements
+
+* Added destructors for named objects to the intermediate representation.

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.12.6
+lastReleaseVersion: 0.12.7

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 0.12.7-dev
+version: 0.12.7
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/src/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.9.6
+
+### Minor Analysis Improvements
+
+* The "non-constant format string" query (`cpp/non-constant-format`) has been converted to a `path-problem` query.
+* The new C/C++ dataflow and taint-tracking libraries (`semmle.code.cpp.dataflow.new.DataFlow` and `semmle.code.cpp.dataflow.new.TaintTracking`) now implicitly assume that dataflow and taint modelled via `DataFlowFunction` and `TaintFunction` always fully overwrite their buffers and thus act as flow barriers. As a result, many dataflow and taint-tracking queries now produce fewer false positives. To remove this assumption and go back to the previous behavior for a given model, one can override the new `isPartialWrite` predicate.
+
 ## 0.9.5
 
 ### Minor Analysis Improvements

cpp/ql/src/change-notes/2024-02-29-non-constant-format-path-query.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The "non-constant format string" query (`cpp/non-constant-format`) has been converted to a `path-problem` query.

cpp/ql/src/change-notes/released/0.9.6.md

@@ -1,4 +1,6 @@
----
-category: minorAnalysis
----
+## 0.9.6
+
+### Minor Analysis Improvements
+
+* The "non-constant format string" query (`cpp/non-constant-format`) has been converted to a `path-problem` query.
 * The new C/C++ dataflow and taint-tracking libraries (`semmle.code.cpp.dataflow.new.DataFlow` and `semmle.code.cpp.dataflow.new.TaintTracking`) now implicitly assume that dataflow and taint modelled via `DataFlowFunction` and `TaintFunction` always fully overwrite their buffers and thus act as flow barriers. As a result, many dataflow and taint-tracking queries now produce fewer false positives. To remove this assumption and go back to the previous behavior for a given model, one can override the new `isPartialWrite` predicate.

cpp/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.9.5
+lastReleaseVersion: 0.9.6

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 0.9.6-dev
+version: 0.9.6
 groups:
   - cpp
   - queries

csharp/ql/campaigns/Solorigate/lib/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.7.10
+
+No user-facing changes.
+
 ## 1.7.9
 
 No user-facing changes.

csharp/ql/campaigns/Solorigate/lib/change-notes/released/1.7.10.md

@@ -0,0 +1,3 @@
+## 1.7.10
+
+No user-facing changes.

csharp/ql/campaigns/Solorigate/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.7.9
+lastReleaseVersion: 1.7.10

csharp/ql/campaigns/Solorigate/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-all
-version: 1.7.10-dev
+version: 1.7.10
 groups:
   - csharp
   - solorigate

csharp/ql/campaigns/Solorigate/src/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.7.10
+
+No user-facing changes.
+
 ## 1.7.9
 
 No user-facing changes.

csharp/ql/campaigns/Solorigate/src/change-notes/released/1.7.10.md

@@ -0,0 +1,3 @@
+## 1.7.10
+
+No user-facing changes.

csharp/ql/campaigns/Solorigate/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.7.9
+lastReleaseVersion: 1.7.10

csharp/ql/campaigns/Solorigate/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-queries
-version: 1.7.10-dev
+version: 1.7.10
 groups:
   - csharp
   - solorigate

csharp/ql/integration-tests/all-platforms/dotnet_build/Program.cs

@@ -1 +1 @@
-Console.WriteLine(args[0]);
+Console.WriteLine($"<arguments>{string.Join(",", args)}</arguments>");

csharp/ql/integration-tests/all-platforms/dotnet_build/test.py

@@ -1,5 +1,16 @@
 from create_database_utils import *
 from diagnostics_test_utils import *
 
-run_codeql_database_create(['dotnet build'], db=None, lang="csharp")
-check_diagnostics()
+def check_build_out(msg, s):
+    if "[build-stdout] " + msg not in s:
+        raise Exception("The C# tracer did not interpret the dotnet path-to-application command correctly.")
+
+run_codeql_database_create(['dotnet build'], test_db="test1-db", lang="csharp")
+check_diagnostics(test_db="test1-db")
+
+# This test checks that we don't inject any flags when running the application using `dotnet`
+my_dir = "my_program"
+my_abs_path = os.path.abspath(f"{my_dir}/dotnet_build.dll")
+s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test1-db', 'dotnet build -o my_program', f'dotnet {my_abs_path} build is not a subcommand'], "test2-db", "csharp")
+check_build_out("<arguments>build,is,not,a,subcommand</arguments>", s)
+check_diagnostics(test_db="test2-db")

csharp/ql/integration-tests/all-platforms/dotnet_run/test.py

@@ -1,65 +1,57 @@
 from create_database_utils import *
 from diagnostics_test_utils import *
 
-def run_codeql_database_create_stdout(args, dbname):
-    stdout = open(dbname + "file.txt", 'w+')
-    run_codeql_database_create(args, test_db=dbname, db=None, stdout=stdout, lang="csharp")
-    stdout.seek(0)
-    s = stdout.read()
-    stdout.close()
-    return s
-
 def check_build_out(msg, s):
     if "[build-stdout] " + msg not in s:
-        raise Exception("The C# extractor did not interpret the 'dotnet run' command correctly")
+        raise Exception("The C# tracer did not interpret the 'dotnet run' command correctly")
 
 # no arguments
-s = run_codeql_database_create_stdout(['dotnet run'], "test-db")
+s = run_codeql_database_create_stdout(['dotnet run'], "test-db", "csharp")
 check_build_out("Default reply", s)
 check_diagnostics()
 
 # no arguments, but `--`
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test-db', 'dotnet run --'], "test2-db")
+s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test-db', 'dotnet run --'], "test2-db", "csharp")
 check_build_out("Default reply", s)
 check_diagnostics(test_db="test2-db")
 
 # one argument, no `--`
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test2-db', 'dotnet run hello'], "test3-db")
+s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test2-db', 'dotnet run hello'], "test3-db", "csharp")
 check_build_out("Default reply", s)
 check_diagnostics(test_db="test3-db")
 
 # one argument, but `--`
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test3-db', 'dotnet run -- hello'], "test4-db")
+s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test3-db', 'dotnet run -- hello'], "test4-db", "csharp")
 check_build_out("Default reply", s)
 check_diagnostics(test_db="test4-db")
 
 # two arguments, no `--`
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test4-db', 'dotnet run hello world'], "test5-db")
+s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test4-db', 'dotnet run hello world'], "test5-db", "csharp")
 check_build_out("hello, world", s)
 check_diagnostics(test_db="test5-db")
 
 # two arguments, and `--`
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test5-db', 'dotnet run -- hello world'], "test6-db")
+s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test5-db', 'dotnet run -- hello world'], "test6-db", "csharp")
 check_build_out("hello, world", s)
 check_diagnostics(test_db="test6-db")
 
 # shared compilation enabled; tracer should override by changing the command
 # to `dotnet run -p:UseSharedCompilation=true -p:UseSharedCompilation=false -- hello world`
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test6-db', 'dotnet run -p:UseSharedCompilation=true -- hello world'], "test7-db")
+s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test6-db', 'dotnet run -p:UseSharedCompilation=true -- hello world'], "test7-db", "csharp")
 check_build_out("hello, world", s)
 check_diagnostics(test_db="test7-db")
 
 # option passed into `dotnet run`
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test7-db', 'dotnet build', 'dotnet run --no-build hello world'], "test8-db")
+s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test7-db', 'dotnet build', 'dotnet run --no-build hello world'], "test8-db", "csharp")
 check_build_out("hello, world", s)
 check_diagnostics(test_db="test8-db")
 
 # two arguments, no '--' (first argument quoted)
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test8-db', 'dotnet run "hello world part1" part2'], "test9-db")
+s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test8-db', 'dotnet run "hello world part1" part2'], "test9-db", "csharp")
 check_build_out("hello world part1, part2", s)
 check_diagnostics(test_db="test9-db")
 
 # two arguments, no '--' (second argument quoted) and using dotnet to execute dotnet
-s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test9-db', 'dotnet dotnet run part1 "hello world part2"'], "test10-db")
+s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test9-db', 'dotnet dotnet run part1 "hello world part2"'], "test10-db", "csharp")
 check_build_out("part1, hello world part2", s)
 check_diagnostics(test_db="test10-db")

csharp/ql/lib/CHANGELOG.md

@@ -1,3 +1,17 @@
+## 0.8.10
+
+### Major Analysis Improvements
+
+* Improved support for flow through captured variables that properly adheres to inter-procedural control flow.
+* We no longer make use of CodeQL database stats, which may affect join-orders in custom queries. It is therefore recommended to test performance of custom queries after upgrading to this version.
+
+### Minor Analysis Improvements
+
+* C# 12: Add QL library support (`ExperimentalAttribute`) for the experimental attribute.
+* C# 12: Add extractor and QL library support for `ref readonly` parameters.
+* C#: The table `expr_compiler_generated` has been deleted and its content has been added to `compiler_generated`.
+* Data flow via get only properties like `public object Obj { get; }` is now captured by the data flow library.
+
 ## 0.8.9
 
 ### Minor Analysis Improvements

csharp/ql/lib/change-notes/2024-02-21-getonly-properties.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* Data flow via get only properties like `public object Obj { get; }` is now captured by the data flow library.

csharp/ql/lib/change-notes/2024-02-22-no-db-stats.md

@@ -1,4 +0,0 @@
----
-category: majorAnalysis
----
-* We no longer make use of CodeQL database stats, which may affect join-orders in custom queries. It is therefore recommended to test performance of custom queries after upgrading to this version.

csharp/ql/lib/change-notes/2024-02-23-compiler-generated.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* C#: The table `expr_compiler_generated` has been deleted and its content has been added to `compiler_generated`.

csharp/ql/lib/change-notes/2024-02-26-variable-capture-flow.md

@@ -1,4 +0,0 @@
----
-category: majorAnalysis
----
-* Improved support for flow through captured variables that properly adheres to inter-procedural control flow.

csharp/ql/lib/change-notes/2024-02-28-experimental-attribute.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* C# 12: Add QL library support (`ExperimentalAttribute`) for the experimental attribute.

csharp/ql/lib/change-notes/2024-02-28-refreadonly-parameter.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* C# 12: Add extractor and QL library support for `ref readonly` parameters.

csharp/ql/lib/change-notes/released/0.8.10.md

@@ -0,0 +1,13 @@
+## 0.8.10
+
+### Major Analysis Improvements
+
+* Improved support for flow through captured variables that properly adheres to inter-procedural control flow.
+* We no longer make use of CodeQL database stats, which may affect join-orders in custom queries. It is therefore recommended to test performance of custom queries after upgrading to this version.
+
+### Minor Analysis Improvements
+
+* C# 12: Add QL library support (`ExperimentalAttribute`) for the experimental attribute.
+* C# 12: Add extractor and QL library support for `ref readonly` parameters.
+* C#: The table `expr_compiler_generated` has been deleted and its content has been added to `compiler_generated`.
+* Data flow via get only properties like `public object Obj { get; }` is now captured by the data flow library.

csharp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.8.9
+lastReleaseVersion: 0.8.10

csharp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-all
-version: 0.8.10-dev
+version: 0.8.10
 groups: csharp
 dbscheme: semmlecode.csharp.dbscheme
 extractor: csharp

csharp/ql/src/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.8.10
+
+### Minor Analysis Improvements
+
+* Most data flow queries that track flow from *remote* flow sources now use the current *threat model* configuration instead. This doesn't lead to any changes in the produced alerts (as the default configuration is *remote* flow sources) unless the threat model configuration is changed. The changed queries are `cs/code-injection`, `cs/command-line-injection`, `cs/user-controlled-bypass`, `cs/count-untrusted-data-external-api`, `cs/untrusted-data-to-external-api`, `cs/ldap-injection`, `cs/log-forging`, `cs/xml/missing-validation`, `cs/redos`, `cs/regex-injection`, `cs/resource-injection`, `cs/sql-injection`, `cs/path-injection`, `cs/unsafe-deserialization-untrusted-input`, `cs/web/unvalidated-url-redirection`, `cs/xml/insecure-dtd-handling`, `cs/xml/xpath-injection`, `cs/web/xss`, and `cs/uncontrolled-format-string`.
+
 ## 0.8.9
 
 ### Minor Analysis Improvements

csharp/ql/src/change-notes/released/0.8.10.md

@@ -1,4 +1,5 @@
----
-category: minorAnalysis
----
-* Most data flow queries that track flow from *remote* flow sources now use the current *threat model* configuration instead. This doesn't lead to any changes in the produced alerts (as the default configuration is *remote* flow sources) unless the threat model configuration is changed. The changed queries are `cs/code-injection`, `cs/command-line-injection`, `cs/user-controlled-bypass`, `cs/count-untrusted-data-external-api`, `cs/untrusted-data-to-external-api`, `cs/ldap-injection`, `cs/log-forging`, `cs/xml/missing-validation`, `cs/redos`, `cs/regex-injection`, `cs/resource-injection`, `cs/sql-injection`, `cs/path-injection`, `cs/unsafe-deserialization-untrusted-input`, `cs/web/unvalidated-url-redirection`, `cs/xml/insecure-dtd-handling`, `cs/xml/xpath-injection`, `cs/web/xss`, and `cs/uncontrolled-format-string`.
+## 0.8.10
+
+### Minor Analysis Improvements
+
+* Most data flow queries that track flow from *remote* flow sources now use the current *threat model* configuration instead. This doesn't lead to any changes in the produced alerts (as the default configuration is *remote* flow sources) unless the threat model configuration is changed. The changed queries are `cs/code-injection`, `cs/command-line-injection`, `cs/user-controlled-bypass`, `cs/count-untrusted-data-external-api`, `cs/untrusted-data-to-external-api`, `cs/ldap-injection`, `cs/log-forging`, `cs/xml/missing-validation`, `cs/redos`, `cs/regex-injection`, `cs/resource-injection`, `cs/sql-injection`, `cs/path-injection`, `cs/unsafe-deserialization-untrusted-input`, `cs/web/unvalidated-url-redirection`, `cs/xml/insecure-dtd-handling`, `cs/xml/xpath-injection`, `cs/web/xss`, and `cs/uncontrolled-format-string`.

csharp/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.8.9
+lastReleaseVersion: 0.8.10

csharp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-queries
-version: 0.8.10-dev
+version: 0.8.10
 groups:
   - csharp
   - queries

csharp/tools/tracing-config.lua

@@ -23,6 +23,10 @@ function RegisterExtractorPack(id)
                not isDotnetPath(arg)
     end
 
+    local function isPathToExecutable(path)
+        return path:match('%.exe$') or path:match('%.dll')
+    end
+
     function DotnetMatcherBuild(compilerName, compilerPath, compilerArguments,
                                 _languageId)
         if not isDotnet(compilerName) then
@@ -56,8 +60,16 @@ function RegisterExtractorPack(id)
             NativeArgumentsToArgv(compilerArguments.nativeArgumentPointer)
         end
         for i, arg in ipairs(argv) do
+            -- if dotnet is being used to execute any application except dotnet itself, we should
+            -- not inject any flags.
+            if not match and isPathToExecutable(arg) and not isDotnetPath(arg) then
+                Log(1, 'Execute a .NET application usage detected')
+                Log(1, 'Dotnet path-to-application detected: %s', arg)
+                break
+            end
             if isPossibleDotnetSubcommand(arg) then
-                if (not match) and inSubCommandPosition then
+                if not match and inSubCommandPosition then
+                    Log(1, 'Execute a .NET SDK command usage detected')
                     Log(1, 'Dotnet subcommand detected: %s', arg)
                 end
                 -- only respond to strings that look like sub-command names if we have not yet
@@ -85,7 +97,7 @@ function RegisterExtractorPack(id)
             end
             -- for `dotnet test`, we should not append `-p:UseSharedCompilation=false` to the command line
             -- if an `exe` or `dll` is passed as an argument as the call is forwarded to vstest.
-            if testMatch and (arg:match('%.exe$') or arg:match('%.dll')) then
+            if testMatch and isPathToExecutable(arg) then
                 match = false
                 break
             end

docs/codeql/codeql-language-guides/analyzing-data-flow-in-java.rst

@@ -27,7 +27,13 @@ Local data flow is data flow within a single method or callable. Local data flow
 Using local data flow
 ~~~~~~~~~~~~~~~~~~~~~
 
-The local data flow library is in the module ``DataFlow``, which defines the class ``Node`` denoting any element that data can flow through. ``Node``\ s are divided into expression nodes (``ExprNode``) and parameter nodes (``ParameterNode``). You can map between data flow nodes and expressions/parameters using the member predicates ``asExpr`` and ``asParameter``:
+To use the data flow library you need the following import:
+
+.. code-block:: ql
+
+   import semmle.code.java.dataflow.DataFlow
+
+The ``DataFlow`` module defines the class ``Node`` denoting any element that data can flow through. ``Node``\ s are divided into expression nodes (``ExprNode``) and parameter nodes (``ParameterNode``). You can map between data flow nodes and expressions/parameters using the member predicates ``asExpr`` and ``asParameter``:
 
 .. code-block:: ql
 
@@ -75,7 +81,14 @@ Local taint tracking extends local data flow by including non-value-preserving f
 
 If ``x`` is a tainted string then ``y`` is also tainted.
 
-The local taint tracking library is in the module ``TaintTracking``. Like local data flow, a predicate ``localTaintStep(DataFlow::Node nodeFrom, DataFlow::Node nodeTo)`` holds if there is an immediate taint propagation edge from the node ``nodeFrom`` to the node ``nodeTo``. You can apply the predicate recursively by using the ``+`` and ``*`` operators, or by using the predefined recursive predicate ``localTaint``, which is equivalent to ``localTaintStep*``.
+
+To use the taint tracking library you need the following import:
+
+.. code-block:: ql
+
+   import semmle.code.java.dataflow.TaintTracking
+
+Like local data flow, a predicate ``localTaintStep(DataFlow::Node nodeFrom, DataFlow::Node nodeTo)`` holds if there is an immediate taint propagation edge from the node ``nodeFrom`` to the node ``nodeTo``. You can apply the predicate recursively by using the ``+`` and ``*`` operators, or by using the predefined recursive predicate ``localTaint``, which is equivalent to ``localTaintStep*``.
 
 For example, you can find taint propagation from a parameter ``source`` to an expression ``sink`` in zero or more local steps:
 

go/extractor/autobuilder/autobuilder.go

@@ -70,12 +70,33 @@ func tryBuild(cmd string, args ...string) bool {
 	return res && (!CheckExtracted || checkExtractorRun())
 }
 
-// Autobuild attempts to detect build system and run the corresponding command.
-func Autobuild() bool {
-	return tryBuildIfExists("Makefile", "make") ||
-		tryBuildIfExists("makefile", "make") ||
-		tryBuildIfExists("GNUmakefile", "make") ||
-		tryBuildIfExists("build.ninja", "ninja") ||
-		tryBuildIfExists("build", "./build") ||
-		tryBuildIfExists("build.sh", "./build.sh")
+// If a project is accompanied by a build script (such as a makefile), then we try executing such
+// build scripts to build the project. This type represents pairs of script names to check for
+// and the names of corresponding build tools to invoke if those scripts exist.
+type BuildScript struct {
+	Tool     string // The name of the command to execute if the build script exists
+	Filename string // The name of the build script to check for
+}
+
+// An array of build scripts to check for and corresponding commands that we can execute
+// if they exist.
+var BuildScripts = []BuildScript{
+	{Tool: "make", Filename: "Makefile"},
+	{Tool: "make", Filename: "makefile"},
+	{Tool: "make", Filename: "GNUmakefile"},
+	{Tool: "ninja", Filename: "build.ninja"},
+	{Tool: "./build", Filename: "build"},
+	{Tool: "./build.sh", Filename: "build.sh"},
+}
+
+// Autobuild attempts to detect build systems based on the presence of build scripts from the
+// list in `BuildScripts` and run the corresponding command. This may invoke zero or more
+// build scripts in the order given by `BuildScripts`.
+func Autobuild() bool {
+	for _, script := range BuildScripts {
+		if tryBuildIfExists(script.Filename, script.Tool) {
+			return true
+		}
+	}
+	return false
 }

go/extractor/autobuilder/build-environment.go

@@ -12,7 +12,7 @@ import (
 )
 
 const minGoVersion = "1.11"
-const maxGoVersion = "1.21"
+const maxGoVersion = "1.22"
 
 type versionInfo struct {
 	goModVersion      string // The version of Go found in the go directive in the `go.mod` file.
@@ -267,15 +267,22 @@ func outputEnvironmentJson(version string) {
 // Get the version of Go to install and output it to stdout as json.
 func IdentifyEnvironment() {
 	var v versionInfo
-	buildInfo := project.GetBuildInfo(false)
-	goVersionInfo := project.TryReadGoDirective(buildInfo)
-	v.goModVersion, v.goModVersionFound = goVersionInfo.Version, goVersionInfo.Found
+	workspaces := project.GetWorkspaceInfo(false)
 
+	// Remove temporary extractor files (e.g. auto-generated go.mod files) when we are done
+	defer project.RemoveTemporaryExtractorFiles()
+
+	// Find the greatest Go version required by any of the workspaces.
+	greatestGoVersion := project.RequiredGoVersion(&workspaces)
+	v.goModVersion, v.goModVersionFound = greatestGoVersion.Version, greatestGoVersion.Found
+
+	// Find which, if any, version of Go is installed on the system already.
 	v.goEnvVersionFound = toolchain.IsInstalled()
 	if v.goEnvVersionFound {
 		v.goEnvVersion = toolchain.GetEnvGoVersion()[2:]
 	}
 
+	// Determine which version of Go we should recommend to install.
 	msg, versionToInstall := getVersionToInstall(v)
 	log.Println(msg)
 

go/extractor/cli/go-autobuilder/go-autobuilder.go

@@ -3,7 +3,6 @@ package main
 import (
 	"fmt"
 	"log"
-	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -56,80 +55,6 @@ Build behavior:
 	fmt.Fprintf(os.Stderr, "Usage:\n\n  %s\n", os.Args[0])
 }
 
-// Returns the current Go version in semver format, e.g. v1.14.4
-func getEnvGoSemVer() string {
-	goVersion := toolchain.GetEnvGoVersion()
-	if !strings.HasPrefix(goVersion, "go") {
-		log.Fatalf("Expected 'go version' output of the form 'go1.2.3'; got '%s'", goVersion)
-	}
-	// Go versions don't follow the SemVer format, but the only exception we normally care about
-	// is release candidates; so this is a horrible hack to convert e.g. `go1.22rc1` into `go1.22-rc1`
-	// which is compatible with the SemVer specification
-	rcIndex := strings.Index(goVersion, "rc")
-	if rcIndex != -1 {
-		return semver.Canonical("v"+goVersion[2:rcIndex]) + "-" + goVersion[rcIndex:]
-	} else {
-		return semver.Canonical("v" + goVersion[2:])
-	}
-}
-
-// Returns the import path of the package being built, or "" if it cannot be determined.
-func getImportPath() (importpath string) {
-	importpath = os.Getenv("LGTM_INDEX_IMPORT_PATH")
-	if importpath == "" {
-		repourl := os.Getenv("SEMMLE_REPO_URL")
-		if repourl == "" {
-			githubrepo := os.Getenv("GITHUB_REPOSITORY")
-			if githubrepo == "" {
-				log.Printf("Unable to determine import path, as neither LGTM_INDEX_IMPORT_PATH nor GITHUB_REPOSITORY is set\n")
-				return ""
-			} else {
-				importpath = "github.com/" + githubrepo
-			}
-		} else {
-			importpath = getImportPathFromRepoURL(repourl)
-			if importpath == "" {
-				log.Printf("Failed to determine import path from SEMMLE_REPO_URL '%s'\n", repourl)
-				return
-			}
-		}
-	}
-	log.Printf("Import path is '%s'\n", importpath)
-	return
-}
-
-// Returns the import path of the package being built from `repourl`, or "" if it cannot be
-// determined.
-func getImportPathFromRepoURL(repourl string) string {
-	// check for scp-like URL as in "git@github.com:github/codeql-go.git"
-	shorturl := regexp.MustCompile(`^([^@]+@)?([^:]+):([^/].*?)(\.git)?$`)
-	m := shorturl.FindStringSubmatch(repourl)
-	if m != nil {
-		return m[2] + "/" + m[3]
-	}
-
-	// otherwise parse as proper URL
-	u, err := url.Parse(repourl)
-	if err != nil {
-		log.Fatalf("Malformed repository URL '%s'\n", repourl)
-	}
-
-	if u.Scheme == "file" {
-		// we can't determine import paths from file paths
-		return ""
-	}
-
-	if u.Hostname() == "" || u.Path == "" {
-		return ""
-	}
-
-	host := u.Hostname()
-	path := u.Path
-	// strip off leading slashes and trailing `.git` if present
-	path = regexp.MustCompile(`^/+|\.git$`).ReplaceAllString(path, "")
-	return host + "/" + path
-}
-
 func restoreRepoLayout(fromDir string, dirEntries []string, scratchDirName string, toDir string) {
 	for _, dirEntry := range dirEntries {
 		if dirEntry != scratchDirName {
@@ -177,8 +102,8 @@ func getSourceDir() string {
 }
 
 // fixGoVendorIssues fixes issues with go vendor for go version >= 1.14
-func fixGoVendorIssues(buildInfo *project.BuildInfo, goModVersionFound bool) {
-	if buildInfo.ModMode == project.ModVendor {
+func fixGoVendorIssues(workspace *project.GoWorkspace, goModVersionFound bool) {
+	if workspace.ModMode == project.ModVendor {
 		// fix go vendor issues with go versions >= 1.14 when no go version is specified in the go.mod
 		// if this is the case, and dependencies were vendored with an old go version (and therefore
 		// do not contain a '## explicit' annotation, the go command will fail and refuse to do any
@@ -186,7 +111,7 @@ func fixGoVendorIssues(buildInfo *project.BuildInfo, goModVersionFound bool) {
 		//
 		// we work around this by adding an explicit go version of 1.13, which is the last version
 		// where this is not an issue
-		if buildInfo.DepMode == project.GoGetWithModules {
+		if workspace.DepMode == project.GoGetWithModules {
 			if !goModVersionFound {
 				// if the go.mod does not contain a version line
 				modulesTxt, err := os.ReadFile("vendor/modules.txt")
@@ -197,7 +122,7 @@ func fixGoVendorIssues(buildInfo *project.BuildInfo, goModVersionFound bool) {
 					log.Println("Adding a version directive to the go.mod file as the modules.txt does not have explicit annotations")
 					if !addVersionToMod("1.13") {
 						log.Println("Failed to add a version to the go.mod file to fix explicitly required package bug; not using vendored dependencies")
-						buildInfo.ModMode = project.ModMod
+						workspace.ModMode = project.ModMod
 					}
 				}
 			}
@@ -206,9 +131,9 @@ func fixGoVendorIssues(buildInfo *project.BuildInfo, goModVersionFound bool) {
 }
 
 // Determines whether the project needs a GOPATH set up
-func getNeedGopath(buildInfo project.BuildInfo, importpath string) bool {
+func getNeedGopath(workspace project.GoWorkspace, importpath string) bool {
 	needGopath := true
-	if buildInfo.DepMode == project.GoGetWithModules {
+	if workspace.DepMode == project.GoGetWithModules {
 		needGopath = false
 	}
 	// if `LGTM_INDEX_NEED_GOPATH` is set, it overrides the value for `needGopath` inferred above
@@ -229,44 +154,46 @@ func getNeedGopath(buildInfo project.BuildInfo, importpath string) bool {
 }
 
 // Try to update `go.mod` and `go.sum` if the go version is >= 1.16.
-func tryUpdateGoModAndGoSum(buildInfo project.BuildInfo) {
+func tryUpdateGoModAndGoSum(workspace project.GoWorkspace) {
 	// Go 1.16 and later won't automatically attempt to update go.mod / go.sum during package loading, so try to update them here:
-	if buildInfo.ModMode != project.ModVendor && buildInfo.DepMode == project.GoGetWithModules && semver.Compare(getEnvGoSemVer(), "v1.16") >= 0 {
-		// stat go.mod and go.sum
-		goModPath := filepath.Join(buildInfo.BaseDir, "go.mod")
-		beforeGoModFileInfo, beforeGoModErr := os.Stat(goModPath)
-		if beforeGoModErr != nil {
-			log.Println("Failed to stat go.mod before running `go mod tidy -e`")
-		}
-
-		goSumPath := filepath.Join(buildInfo.BaseDir, "go.sum")
-		beforeGoSumFileInfo, beforeGoSumErr := os.Stat(goSumPath)
-
-		// run `go mod tidy -e`
-		cmd := exec.Command("go", "mod", "tidy", "-e")
-		cmd.Dir = buildInfo.BaseDir
-		res := util.RunCmd(cmd)
-
-		if !res {
-			log.Println("Failed to run `go mod tidy -e`")
-		} else {
-			if beforeGoModFileInfo != nil {
-				afterGoModFileInfo, afterGoModErr := os.Stat(goModPath)
-				if afterGoModErr != nil {
-					log.Println("Failed to stat go.mod after running `go mod tidy -e`")
-				} else if afterGoModFileInfo.ModTime().After(beforeGoModFileInfo.ModTime()) {
-					// if go.mod has been changed then notify the user
-					log.Println("We have run `go mod tidy -e` and it altered go.mod. You may wish to check these changes into version control. ")
-				}
+	if workspace.ModMode != project.ModVendor && workspace.DepMode == project.GoGetWithModules && semver.Compare(toolchain.GetEnvGoSemVer(), "v1.16") >= 0 {
+		for _, goMod := range workspace.Modules {
+			// stat go.mod and go.sum
+			goModPath := goMod.Path
+			goModDir := filepath.Dir(goModPath)
+			beforeGoModFileInfo, beforeGoModErr := os.Stat(goModPath)
+			if beforeGoModErr != nil {
+				log.Printf("Failed to stat %s before running `go mod tidy -e`\n", goModPath)
 			}
 
-			afterGoSumFileInfo, afterGoSumErr := os.Stat(goSumPath)
-			if afterGoSumErr != nil {
-				log.Println("Failed to stat go.sum after running `go mod tidy -e`")
+			goSumPath := filepath.Join(goModDir, "go.sum")
+			beforeGoSumFileInfo, beforeGoSumErr := os.Stat(goSumPath)
+
+			// run `go mod tidy -e`
+			cmd := toolchain.TidyModule(goModDir)
+			res := util.RunCmd(cmd)
+
+			if !res {
+				log.Printf("Failed to run `go mod tidy -e` in %s\n", goModDir)
 			} else {
-				if beforeGoSumErr != nil || afterGoSumFileInfo.ModTime().After(beforeGoSumFileInfo.ModTime()) {
-					// if go.sum has been changed then notify the user
-					log.Println("We have run `go mod tidy -e` and it altered go.sum. You may wish to check these changes into version control. ")
+				if beforeGoModFileInfo != nil {
+					afterGoModFileInfo, afterGoModErr := os.Stat(goModPath)
+					if afterGoModErr != nil {
+						log.Printf("Failed to stat %s after running `go mod tidy -e`: %s\n", goModPath, afterGoModErr.Error())
+					} else if afterGoModFileInfo.ModTime().After(beforeGoModFileInfo.ModTime()) {
+						// if go.mod has been changed then notify the user
+						log.Println("We have run `go mod tidy -e` and it altered go.mod. You may wish to check these changes into version control. ")
+					}
+				}
+
+				afterGoSumFileInfo, afterGoSumErr := os.Stat(goSumPath)
+				if afterGoSumErr != nil {
+					log.Printf("Failed to stat %s after running `go mod tidy -e`: %s\n", goSumPath, afterGoSumErr.Error())
+				} else {
+					if beforeGoSumErr != nil || afterGoSumFileInfo.ModTime().After(beforeGoSumFileInfo.ModTime()) {
+						// if go.sum has been changed then notify the user
+						log.Println("We have run `go mod tidy -e` and it altered go.sum. You may wish to check these changes into version control. ")
+					}
 				}
 			}
 		}
@@ -406,7 +333,7 @@ func buildWithoutCustomCommands(modMode project.ModMode) bool {
 		log.Println("Build failed, continuing to install dependencies.")
 
 		shouldInstallDependencies = true
-	} else if util.DepErrors("./...", modMode.ArgsForGoVersion(getEnvGoSemVer())...) {
+	} else if util.DepErrors("./...", modMode.ArgsForGoVersion(toolchain.GetEnvGoSemVer())...) {
 		log.Println("Dependencies are still not resolving after the build, continuing to install dependencies.")
 
 		shouldInstallDependencies = true
@@ -449,10 +376,10 @@ func buildWithCustomCommands(inst string) {
 }
 
 // Install dependencies using the given dependency installer mode.
-func installDependencies(buildInfo project.BuildInfo) {
+func installDependencies(workspace project.GoWorkspace) {
 	// automatically determine command to install dependencies
 	var install *exec.Cmd
-	if buildInfo.DepMode == project.Dep {
+	if workspace.DepMode == project.Dep {
 		// set up the dep cache if SEMMLE_CACHE is set
 		cacheDir := os.Getenv("SEMMLE_CACHE")
 		if cacheDir != "" {
@@ -482,47 +409,80 @@ func installDependencies(buildInfo project.BuildInfo) {
 			install = exec.Command("dep", "ensure", "-v")
 		}
 		log.Println("Installing dependencies using `dep ensure`.")
-	} else if buildInfo.DepMode == project.Glide {
+		util.RunCmd(install)
+	} else if workspace.DepMode == project.Glide {
 		install = exec.Command("glide", "install")
 		log.Println("Installing dependencies using `glide install`")
+		util.RunCmd(install)
 	} else {
-		// explicitly set go module support
-		if buildInfo.DepMode == project.GoGetWithModules {
-			os.Setenv("GO111MODULE", "on")
-		} else if buildInfo.DepMode == project.GoGetNoModules {
-			os.Setenv("GO111MODULE", "off")
+		if workspace.Modules == nil {
+			project.InitGoModForLegacyProject(workspace.BaseDir)
+			workspace.Modules = project.LoadGoModules([]string{filepath.Join(workspace.BaseDir, "go.mod")})
 		}
 
-		// get dependencies
-		install = exec.Command("go", "get", "-v", "./...")
-		install.Dir = buildInfo.BaseDir
-		log.Printf("Installing dependencies using `go get -v ./...` in `%s`.\n", buildInfo.BaseDir)
+		// get dependencies for all modules
+		for _, module := range workspace.Modules {
+			path := filepath.Dir(module.Path)
+
+			if util.DirExists(filepath.Join(path, "vendor")) {
+				vendor := toolchain.VendorModule(path)
+				log.Printf("Synchronizing vendor file using `go mod vendor` in %s.\n", path)
+				util.RunCmd(vendor)
+			}
+
+			install = exec.Command("go", "get", "-v", "./...")
+			install.Dir = path
+			log.Printf("Installing dependencies using `go get -v ./...` in `%s`.\n", path)
+			util.RunCmd(install)
+		}
 	}
-	util.RunCmd(install)
 }
 
 // Run the extractor.
-func extract(buildInfo project.BuildInfo) {
+func extract(workspace project.GoWorkspace) bool {
 	extractor, err := util.GetExtractorPath()
 	if err != nil {
 		log.Fatalf("Could not determine path of extractor: %v.\n", err)
 	}
 
 	extractorArgs := []string{}
-	if buildInfo.DepMode == project.GoGetWithModules {
-		extractorArgs = append(extractorArgs, buildInfo.ModMode.ArgsForGoVersion(getEnvGoSemVer())...)
+	if workspace.DepMode == project.GoGetWithModules {
+		extractorArgs = append(extractorArgs, workspace.ModMode.ArgsForGoVersion(toolchain.GetEnvGoSemVer())...)
 	}
-	extractorArgs = append(extractorArgs, "./...")
 
-	log.Printf("Running extractor command '%s %v' from directory '%s'.\n", extractor, extractorArgs, buildInfo.BaseDir)
+	if len(workspace.Modules) == 0 {
+		// There may be no modules if we are using e.g. Dep or Glide
+		extractorArgs = append(extractorArgs, "./...")
+	} else {
+		for _, module := range workspace.Modules {
+			relModPath, relErr := filepath.Rel(workspace.BaseDir, filepath.Dir(module.Path))
+
+			if relErr != nil {
+				log.Printf(
+					"Unable to make module path %s relative to workspace base dir %s: %s\n",
+					filepath.Dir(module.Path), workspace.BaseDir, relErr.Error())
+			} else {
+				if relModPath != "." {
+					extractorArgs = append(extractorArgs, "."+string(os.PathSeparator)+relModPath+"/...")
+				} else {
+					extractorArgs = append(extractorArgs, relModPath+"/...")
+				}
+			}
+		}
+	}
+
+	log.Printf("Running extractor command '%s %v' from directory '%s'.\n", extractor, extractorArgs, workspace.BaseDir)
 	cmd := exec.Command(extractor, extractorArgs...)
-	cmd.Dir = buildInfo.BaseDir
+	cmd.Dir = workspace.BaseDir
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	err = cmd.Run()
 	if err != nil {
-		log.Fatalf("Extraction failed: %s\n", err.Error())
+		log.Printf("Extraction failed for %s: %s\n", workspace.BaseDir, err.Error())
+		return false
 	}
+
+	return true
 }
 
 // Build the project and run the extractor.
@@ -536,75 +496,118 @@ func installDependenciesAndBuild() {
 
 	// determine how to install dependencies and whether a GOPATH needs to be set up before
 	// extraction
-	buildInfo := project.GetBuildInfo(true)
+	workspaces := project.GetWorkspaceInfo(true)
 	if _, present := os.LookupEnv("GO111MODULE"); !present {
 		os.Setenv("GO111MODULE", "auto")
 	}
 
-	goVersionInfo := project.TryReadGoDirective(buildInfo)
+	// Remove temporary extractor files (e.g. auto-generated go.mod files) when we are done
+	defer project.RemoveTemporaryExtractorFiles()
+
+	// If there is only one workspace and it needs a GOPATH set up, which may be the case if
+	// we don't use Go modules, then we move the repository to a temporary directory and set
+	// the GOPATH to it.
+	if len(workspaces) == 1 {
+		workspace := workspaces[0]
+
+		importpath := util.GetImportPath()
+		needGopath := getNeedGopath(workspace, importpath)
+
+		inLGTM := os.Getenv("LGTM_SRC") != "" || os.Getenv("LGTM_INDEX_NEED_GOPATH") != ""
+
+		if inLGTM && needGopath {
+			paths := moveToTemporaryGopath(srcdir, importpath)
+
+			// schedule restoring the contents of newdir to their original location after this function completes:
+			defer restoreRepoLayout(paths.newdir, paths.files, filepath.Base(paths.scratch), srcdir)
+
+			pt := createPathTransformerFile(paths.newdir)
+			defer os.Remove(pt.Name())
+
+			writePathTransformerFile(pt, paths.realSrc, paths.root, paths.newdir)
+			setGopath(paths.root)
+		}
+	}
+
+	// Find the greatest version of Go that is required by the workspaces to check it against the version
+	// of Go that is installed on the system.
+	greatestGoVersion := project.RequiredGoVersion(&workspaces)
 
 	// This diagnostic is not required if the system Go version is 1.21 or greater, since the
 	// Go tooling should install required Go versions as needed.
-	if semver.Compare(getEnvGoSemVer(), "v1.21.0") < 0 && goVersionInfo.Found && semver.Compare("v"+goVersionInfo.Version, getEnvGoSemVer()) > 0 {
-		diagnostics.EmitNewerGoVersionNeeded(getEnvGoSemVer(), "v"+goVersionInfo.Version)
+	if semver.Compare(toolchain.GetEnvGoSemVer(), "v1.21.0") < 0 && greatestGoVersion.Found && semver.Compare("v"+greatestGoVersion.Version, toolchain.GetEnvGoSemVer()) > 0 {
+		diagnostics.EmitNewerGoVersionNeeded(toolchain.GetEnvGoSemVer(), "v"+greatestGoVersion.Version)
 		if val, _ := os.LookupEnv("GITHUB_ACTIONS"); val == "true" {
 			log.Printf(
-				"The go.mod file requires version %s of Go, but version %s is installed. Consider adding an actions/setup-go step to your workflow.\n",
-				"v"+goVersionInfo.Version,
-				getEnvGoSemVer())
+				"A go.mod file requires version %s of Go, but version %s is installed. Consider adding an actions/setup-go step to your workflow.\n",
+				"v"+greatestGoVersion.Version,
+				toolchain.GetEnvGoSemVer())
 		}
 	}
 
-	fixGoVendorIssues(&buildInfo, goVersionInfo.Found)
+	// Track all projects which could not be extracted successfully
+	var unsuccessfulProjects = []string{}
 
-	tryUpdateGoModAndGoSum(buildInfo)
+	// Attempt to extract all workspaces; we will tolerate individual extraction failures here
+	for i, workspace := range workspaces {
+		goVersionInfo := workspace.RequiredGoVersion()
 
-	importpath := getImportPath()
-	needGopath := getNeedGopath(buildInfo, importpath)
+		fixGoVendorIssues(&workspace, goVersionInfo.Found)
 
-	inLGTM := os.Getenv("LGTM_SRC") != "" || os.Getenv("LGTM_INDEX_NEED_GOPATH") != ""
+		tryUpdateGoModAndGoSum(workspace)
 
-	if inLGTM && needGopath {
-		paths := moveToTemporaryGopath(srcdir, importpath)
-
-		// schedule restoring the contents of newdir to their original location after this function completes:
-		defer restoreRepoLayout(paths.newdir, paths.files, filepath.Base(paths.scratch), srcdir)
-
-		pt := createPathTransformerFile(paths.newdir)
-		defer os.Remove(pt.Name())
-
-		writePathTransformerFile(pt, paths.realSrc, paths.root, paths.newdir)
-		setGopath(paths.root)
-	}
-
-	// check whether an explicit dependency installation command was provided
-	inst := util.Getenv("CODEQL_EXTRACTOR_GO_BUILD_COMMAND", "LGTM_INDEX_BUILD_COMMAND")
-	shouldInstallDependencies := false
-	if inst == "" {
-		shouldInstallDependencies = buildWithoutCustomCommands(buildInfo.ModMode)
-	} else {
-		buildWithCustomCommands(inst)
-	}
-
-	if buildInfo.ModMode == project.ModVendor {
-		// test if running `go` with -mod=vendor works, and if it doesn't, try to fallback to -mod=mod
-		// or not set if the go version < 1.14. Note we check this post-build in case the build brings
-		// the vendor directory up to date.
-		if !checkVendor() {
-			buildInfo.ModMode = project.ModMod
-			log.Println("The vendor directory is not consistent with the go.mod; not using vendored dependencies.")
-		}
-	}
-
-	if shouldInstallDependencies {
-		if buildInfo.ModMode == project.ModVendor {
-			log.Printf("Skipping dependency installation because a Go vendor directory was found.")
+		// check whether an explicit dependency installation command was provided
+		inst := util.Getenv("CODEQL_EXTRACTOR_GO_BUILD_COMMAND", "LGTM_INDEX_BUILD_COMMAND")
+		shouldInstallDependencies := false
+		if inst == "" {
+			shouldInstallDependencies = buildWithoutCustomCommands(workspace.ModMode)
 		} else {
-			installDependencies(buildInfo)
+			buildWithCustomCommands(inst)
+		}
+
+		if workspace.ModMode == project.ModVendor {
+			// test if running `go` with -mod=vendor works, and if it doesn't, try to fallback to -mod=mod
+			// or not set if the go version < 1.14. Note we check this post-build in case the build brings
+			// the vendor directory up to date.
+			if !checkVendor() {
+				workspace.ModMode = project.ModMod
+				log.Println("The vendor directory is not consistent with the go.mod; not using vendored dependencies.")
+			}
+		}
+
+		if shouldInstallDependencies {
+			if workspace.ModMode == project.ModVendor {
+				log.Printf("Skipping dependency installation because a Go vendor directory was found.")
+			} else {
+				installDependencies(workspace)
+			}
+		}
+
+		workspaces[i].Extracted = extract(workspace)
+
+		if !workspaces[i].Extracted {
+			unsuccessfulProjects = append(unsuccessfulProjects, workspace.BaseDir)
 		}
 	}
 
-	extract(buildInfo)
+	// If all projects could not be extracted successfully, we fail the overall extraction.
+	if len(unsuccessfulProjects) == len(workspaces) {
+		log.Fatalln("Extraction failed for all discovered Go projects.")
+	}
+
+	// If there is at least one project that could not be extracted successfully,
+	// emit a diagnostic that reports which projects we could not extract successfully.
+	// We only consider this a warning, since there may be test projects etc. which
+	// do not matter if they cannot be extracted successfully.
+	if len(unsuccessfulProjects) > 0 {
+		log.Printf(
+			"Warning: extraction failed for %d project(s): %s\n",
+			len(unsuccessfulProjects),
+			strings.Join(unsuccessfulProjects, ", "))
+		diagnostics.EmitExtractionFailedForProjects(unsuccessfulProjects)
+	} else {
+		log.Printf("Success: extraction succeeded for all %d discovered project(s).\n", len(workspaces))
+	}
 }
 
 func main() {

go/extractor/diagnostics/diagnostics.go

@@ -493,3 +493,18 @@ func EmitNewerSystemGoRequired(requiredVersion string) {
 		noLocation,
 	)
 }
+
+func EmitExtractionFailedForProjects(path []string) {
+	emitDiagnostic(
+		"go/autobuilder/extraction-failed-for-project",
+		fmt.Sprintf("Unable to extract %d Go projects", len(path)),
+		fmt.Sprintf(
+			"The following %d Go project%s could not be extracted successfully:\n\n`%s`\n",
+			len(path),
+			plural(len(path), "", "s"),
+			strings.Join(path, "`, `")),
+		severityWarning,
+		fullVisibility,
+		noLocation,
+	)
+}

go/extractor/go.mod

@@ -1,6 +1,6 @@
 module github.com/github/codeql-go/extractor
 
-go 1.21
+go 1.22.0
 
 require (
 	golang.org/x/mod v0.15.0

go/extractor/go.work

@@ -0,0 +1,3 @@
+go 1.22.0
+
+use .

go/extractor/go.work.sum

@@ -0,0 +1,5 @@
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/telemetry v0.0.0-20240208230135-b75ee8823808/go.mod h1:KG1lNk5ZFNssSZLrpVb4sMXKMpGwGXOxSG3rnu2gZQQ=

go/extractor/project/project.go

@@ -5,22 +5,104 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"slices"
 	"sort"
 	"strings"
 
 	"github.com/github/codeql-go/extractor/diagnostics"
+	"github.com/github/codeql-go/extractor/toolchain"
 	"github.com/github/codeql-go/extractor/util"
+	"golang.org/x/mod/modfile"
 	"golang.org/x/mod/semver"
 )
 
-func getDirs(paths []string) []string {
-	dirs := make([]string, len(paths))
-	for i, path := range paths {
-		dirs[i] = filepath.Dir(path)
-	}
-	return dirs
+// DependencyInstallerMode is an enum describing how dependencies should be installed
+type DependencyInstallerMode int
+
+const (
+	// GoGetNoModules represents dependency installation using `go get` without modules
+	GoGetNoModules DependencyInstallerMode = iota
+	// GoGetWithModules represents dependency installation using `go get` with modules
+	GoGetWithModules
+	// Dep represent dependency installation using `dep ensure`
+	Dep
+	// Glide represents dependency installation using `glide install`
+	Glide
+)
+
+// Represents information about a `go.mod` file: this is at least the path to the `go.mod` file,
+// plus the parsed contents of the file, if available.
+type GoModule struct {
+	Path   string        // The path to the `go.mod` file
+	Module *modfile.File // The parsed contents of the `go.mod` file
 }
 
+// Represents information about a Go project workspace: this may either be a folder containing
+// a `go.work` file or a collection of `go.mod` files.
+type GoWorkspace struct {
+	BaseDir       string                  // The base directory for this workspace
+	WorkspaceFile *modfile.WorkFile       // The `go.work` file for this workspace
+	Modules       []*GoModule             // A list of `go.mod` files
+	DepMode       DependencyInstallerMode // A value indicating how to install dependencies for this workspace
+	ModMode       ModMode                 // A value indicating which module mode to use for this workspace
+	Extracted     bool                    // A value indicating whether this workspace was extracted successfully
+}
+
+// Represents a nullable version string.
+type GoVersionInfo struct {
+	// The version string, if any
+	Version string
+	// A value indicating whether a version string was found
+	Found bool
+}
+
+// Determines the version of Go that is required by this workspace. This is, in order of preference:
+// 1. The Go version specified in the `go.work` file, if any.
+// 2. The greatest Go version specified in any `go.mod` file, if any.
+func (workspace *GoWorkspace) RequiredGoVersion() GoVersionInfo {
+	if workspace.WorkspaceFile != nil && workspace.WorkspaceFile.Go != nil {
+		// If we have parsed a `go.work` file, return the version number from it.
+		return GoVersionInfo{Version: workspace.WorkspaceFile.Go.Version, Found: true}
+	} else if workspace.Modules != nil && len(workspace.Modules) > 0 {
+		// Otherwise, if we have `go.work` files, find the greatest Go version in those.
+		var greatestVersion string = ""
+		for _, module := range workspace.Modules {
+			if module.Module != nil && module.Module.Go != nil {
+				// If we have parsed the file, retrieve the version number we have already obtained.
+				if greatestVersion == "" || semver.Compare("v"+module.Module.Go.Version, "v"+greatestVersion) > 0 {
+					greatestVersion = module.Module.Go.Version
+				}
+			} else {
+				modVersion := tryReadGoDirective(module.Path)
+				if modVersion.Found && (greatestVersion == "" || semver.Compare("v"+modVersion.Version, "v"+greatestVersion) > 0) {
+					greatestVersion = modVersion.Version
+				}
+			}
+		}
+
+		// If we have found some version, return it.
+		if greatestVersion != "" {
+			return GoVersionInfo{Version: greatestVersion, Found: true}
+		}
+	}
+
+	return GoVersionInfo{Version: "", Found: false}
+}
+
+// Finds the greatest Go version required by any of the given `workspaces`.
+// Returns a `GoVersionInfo` value with `Found: false` if no version information is available.
+func RequiredGoVersion(workspaces *[]GoWorkspace) GoVersionInfo {
+	greatestGoVersion := GoVersionInfo{Version: "", Found: false}
+	for _, workspace := range *workspaces {
+		goVersionInfo := workspace.RequiredGoVersion()
+		if goVersionInfo.Found && (!greatestGoVersion.Found || semver.Compare("v"+goVersionInfo.Version, "v"+greatestGoVersion.Version) > 0) {
+			greatestGoVersion = goVersionInfo
+		}
+	}
+	return greatestGoVersion
+}
+
+// Determines whether any of the directory paths in the input are nested.
 func checkDirsNested(inputDirs []string) (string, bool) {
 	// replace "." with "" so that we can check if all the paths are nested
 	dirs := make([]string, len(inputDirs))
@@ -42,72 +124,376 @@ func checkDirsNested(inputDirs []string) (string, bool) {
 	return dirs[0], true
 }
 
-// Returns the directory to run the go build in and whether to use a go.mod
-// file.
-func findGoModFiles(emitDiagnostics bool) (baseDir string, useGoMod bool) {
-	goModPaths := util.FindAllFilesWithName(".", "go.mod", "vendor")
-	if len(goModPaths) == 0 {
-		baseDir = "."
-		useGoMod = false
+// A list of files we created that should be removed after we are done.
+var filesToRemove []string = []string{}
+
+// Try to initialize a go.mod file for projects that do not already have one.
+func InitGoModForLegacyProject(path string) {
+	log.Printf("The code in %s seems to be missing a go.mod file. Attempting to initialize one...\n", path)
+
+	modInit := toolchain.InitModule(path)
+
+	if !util.RunCmd(modInit) {
+		log.Printf("Failed to initialize go.mod file for this project.")
 		return
 	}
-	goModDirs := getDirs(goModPaths)
-	if util.AnyGoFilesOutsideDirs(".", goModDirs...) {
+
+	// Add the go.mod file to a list of files we should remove later.
+	filesToRemove = append(filesToRemove, filepath.Join(path, "go.mod"))
+
+	modTidy := toolchain.TidyModule(path)
+	out, err := modTidy.CombinedOutput()
+	log.Println(string(out))
+
+	if err != nil {
+		log.Printf("Failed to determine module requirements for this project.")
+	}
+
+	if strings.Contains(string(out), "is relative, but relative import paths are not supported in module mode") {
+		diagnostics.EmitRelativeImportPaths()
+	}
+}
+
+// Attempts to remove all files that we created.
+func RemoveTemporaryExtractorFiles() {
+	for _, path := range filesToRemove {
+		err := os.Remove(path)
+		if err != nil {
+			log.Printf("Unable to remove file we created at %s: %s\n", path, err.Error())
+		}
+	}
+
+	filesToRemove = []string{}
+}
+
+// Find all go.work files in the working directory and its subdirectories
+func findGoWorkFiles() []string {
+	return util.FindAllFilesWithName(".", "go.work", "vendor")
+}
+
+// Find all go.mod files in the specified directory and its subdirectories
+func findGoModFiles(root string) []string {
+	return util.FindAllFilesWithName(root, "go.mod", "vendor")
+}
+
+// Given a list of `go.mod` file paths, try to parse them all. The resulting array of `GoModule` objects
+// will be the same length as the input array and the objects will contain at least the `go.mod` path.
+// If parsing the corresponding file is successful, then the parsed contents will also be available.
+func LoadGoModules(goModFilePaths []string) []*GoModule {
+	results := make([]*GoModule, len(goModFilePaths))
+
+	for i, goModFilePath := range goModFilePaths {
+		results[i] = new(GoModule)
+		results[i].Path = goModFilePath
+
+		modFileSrc, err := os.ReadFile(goModFilePath)
+
+		if err != nil {
+			log.Printf("Unable to read %s: %s.\n", goModFilePath, err.Error())
+			continue
+		}
+
+		modFile, err := modfile.ParseLax(goModFilePath, modFileSrc, nil)
+
+		if err != nil {
+			log.Printf("Unable to parse %s: %s.\n", goModFilePath, err.Error())
+			continue
+		}
+
+		results[i].Module = modFile
+	}
+
+	return results
+}
+
+// Given a path to a `go.work` file, this function attempts to parse the `go.work` file. If unsuccessful,
+// we attempt to discover `go.mod` files within subdirectories of the directory containing the `go.work`
+// file ourselves.
+func discoverWorkspace(workFilePath string) GoWorkspace {
+	log.Printf("Loading %s...\n", workFilePath)
+	baseDir := filepath.Dir(workFilePath)
+	workFileSrc, err := os.ReadFile(workFilePath)
+
+	if err != nil {
+		// We couldn't read the `go.work` file for some reason; let's try to find `go.mod` files ourselves
+		log.Printf("Unable to read %s, falling back to finding `go.mod` files manually:\n%s\n", workFilePath, err.Error())
+
+		goModFilePaths := findGoModFiles(baseDir)
+		log.Printf("Discovered the following Go modules in %s:\n%s\n", baseDir, strings.Join(goModFilePaths, "\n"))
+
+		return GoWorkspace{
+			BaseDir: baseDir,
+			Modules: LoadGoModules(goModFilePaths),
+			DepMode: GoGetWithModules,
+			ModMode: getModMode(GoGetWithModules, baseDir),
+		}
+	}
+
+	workFile, err := modfile.ParseWork(workFilePath, workFileSrc, nil)
+
+	if err != nil {
+		// The `go.work` file couldn't be parsed for some reason; let's try to find `go.mod` files ourselves
+		log.Printf("Unable to parse %s, falling back to finding `go.mod` files manually:\n%s\n", workFilePath, err.Error())
+
+		goModFilePaths := findGoModFiles(baseDir)
+		log.Printf("Discovered the following Go modules in %s:\n%s\n", baseDir, strings.Join(goModFilePaths, "\n"))
+
+		return GoWorkspace{
+			BaseDir: baseDir,
+			Modules: LoadGoModules(goModFilePaths),
+			DepMode: GoGetWithModules,
+			ModMode: getModMode(GoGetWithModules, baseDir),
+		}
+	}
+
+	// Get the paths of all of the `go.mod` files that we read from the `go.work` file.
+	goModFilePaths := make([]string, len(workFile.Use))
+
+	for i, use := range workFile.Use {
+		if filepath.IsAbs(use.Path) {
+			// TODO: This case might be problematic for some other logic (e.g. stray file detection)
+			goModFilePaths[i] = filepath.Join(use.Path, "go.mod")
+		} else {
+			goModFilePaths[i] = filepath.Join(filepath.Dir(workFilePath), use.Path, "go.mod")
+		}
+	}
+
+	log.Printf("%s uses the following Go modules:\n%s\n", workFilePath, strings.Join(goModFilePaths, "\n"))
+
+	return GoWorkspace{
+		BaseDir:       baseDir,
+		WorkspaceFile: workFile,
+		Modules:       LoadGoModules(goModFilePaths),
+		DepMode:       GoGetWithModules,
+		ModMode:       ModReadonly, // Workspaces only support "readonly"
+	}
+}
+
+// Analyse the working directory to discover workspaces.
+func discoverWorkspaces(emitDiagnostics bool) []GoWorkspace {
+	// Try to find any `go.work` files which may exist in the working directory.
+	goWorkFiles := findGoWorkFiles()
+
+	if len(goWorkFiles) == 0 {
+		// There is no `go.work` file. Find all `go.mod` files in the working directory.
+		log.Println("Found no go.work files in the workspace; looking for go.mod files...")
+
+		goModFiles := findGoModFiles(".")
+
+		// Return a separate workspace for each `go.mod` file that we found.
+		results := make([]GoWorkspace, len(goModFiles))
+
+		for i, goModFile := range goModFiles {
+			results[i] = GoWorkspace{
+				BaseDir: filepath.Dir(goModFile),
+				Modules: LoadGoModules([]string{goModFile}),
+				DepMode: GoGetWithModules,
+				ModMode: getModMode(GoGetWithModules, filepath.Dir(goModFile)),
+			}
+		}
+
+		return results
+	} else {
+		// We have found `go.work` files, try to load them all.
+		log.Printf("Found go.work file(s) in: %s.\n", strings.Join(goWorkFiles, ", "))
+
+		if emitDiagnostics {
+			diagnostics.EmitGoWorkFound(goWorkFiles)
+		}
+
+		results := make([]GoWorkspace, len(goWorkFiles))
+		for i, workFilePath := range goWorkFiles {
+			results[i] = discoverWorkspace(workFilePath)
+		}
+
+		// Add all stray `go.mod` files (i.e. those not referenced by `go.work` files)
+		// as separate workspaces.
+		goModFiles := findGoModFiles(".")
+
+		for _, goModFile := range goModFiles {
+			// Check to see whether we already have this module file under an existing workspace.
+			found := false
+			for _, workspace := range results {
+				if workspace.Modules == nil {
+					break
+				}
+
+				for _, module := range workspace.Modules {
+					if module.Path == goModFile {
+						found = true
+						break
+					}
+				}
+
+				if found {
+					break
+				}
+			}
+
+			// If not, add it to the array.
+			if !found {
+				log.Printf("Module %s is not referenced by any go.work file; adding it separately.\n", goModFile)
+				results = append(results, GoWorkspace{
+					BaseDir: filepath.Dir(goModFile),
+					Modules: LoadGoModules([]string{goModFile}),
+					DepMode: GoGetWithModules,
+					ModMode: getModMode(GoGetWithModules, filepath.Dir(goModFile)),
+				})
+			}
+		}
+
+		return results
+	}
+}
+
+// Discovers Go workspaces in the current working directory.
+// Returns an array of Go workspaces and the total number of module files which we discovered.
+func getBuildRoots(emitDiagnostics bool) (goWorkspaces []GoWorkspace, totalModuleFiles int) {
+	goWorkspaces = discoverWorkspaces(emitDiagnostics)
+
+	// Determine the total number of `go.mod` files that we discovered.
+	totalModuleFiles = 0
+
+	for _, goWorkspace := range goWorkspaces {
+		totalModuleFiles += len(goWorkspace.Modules)
+	}
+
+	// If there are no `go.mod` files at all, create one in line with https://go.dev/blog/migrating-to-go-modules
+	if totalModuleFiles == 0 {
+		// Check for other, legacy package managers
+		if util.FileExists("Gopkg.toml") {
+			if emitDiagnostics {
+				diagnostics.EmitGopkgTomlFound()
+			}
+			log.Println("Found Gopkg.toml, using dep instead of go get")
+			goWorkspaces = []GoWorkspace{{
+				BaseDir: ".",
+				DepMode: Dep,
+				ModMode: ModUnset,
+			}}
+			totalModuleFiles = 0
+			return
+		}
+
+		if util.FileExists("glide.yaml") {
+			if emitDiagnostics {
+				diagnostics.EmitGlideYamlFound()
+			}
+			log.Println("Found glide.yaml, using Glide instead of go get")
+			goWorkspaces = []GoWorkspace{{
+				BaseDir: ".",
+				DepMode: Glide,
+				ModMode: ModUnset,
+			}}
+			totalModuleFiles = 0
+			return
+		}
+
+		// If we have no `go.mod` files, then the project appears to be a legacy project without
+		// a `go.mod` file. Check that there are actually Go source files before initializing a module
+		// so that we correctly fail the extraction later.
+		if !util.FindGoFiles(".") {
+			goWorkspaces = []GoWorkspace{{
+				BaseDir: ".",
+				DepMode: GoGetNoModules,
+				ModMode: ModUnset,
+			}}
+			totalModuleFiles = 0
+			return
+		}
+
+		goWorkspaces = []GoWorkspace{{
+			BaseDir: ".",
+			DepMode: GoGetNoModules,
+			ModMode: getModMode(GoGetWithModules, "."),
+		}}
+		totalModuleFiles = 0
+		return
+	}
+
+	// Get the paths to all `go.mod` files
+	i := 0
+	goModPaths := make([]string, totalModuleFiles)
+
+	for _, goWorkspace := range goWorkspaces {
+		for _, goModule := range goWorkspace.Modules {
+			goModPaths[i] = goModule.Path
+			i++
+		}
+	}
+
+	goModDirs := util.GetParentDirs(goModPaths)
+	straySourceFiles := util.GoFilesOutsideDirs(".", goModDirs...)
+	if len(straySourceFiles) > 0 {
 		if emitDiagnostics {
 			diagnostics.EmitGoFilesOutsideGoModules(goModPaths)
 		}
-		baseDir = "."
-		useGoMod = false
+
+		// We need to initialise Go modules for the stray source files. Our goal is to initialise
+		// as few Go modules as possible, in locations which do not overlap with existing Go
+		// modules.
+		for _, straySourceFile := range straySourceFiles {
+			path := "."
+			components := strings.Split(filepath.Dir(straySourceFile), string(os.PathSeparator))
+
+			for _, component := range components {
+				path = filepath.Join(path, component)
+
+				// Try to initialize a `go.mod` file automatically for the stray source files if
+				// doing so would not place it in a parent directory of an existing `go.mod` file.
+				if !startsWithAnyOf(path, goModDirs) {
+					goWorkspaces = append(goWorkspaces, GoWorkspace{
+						BaseDir: path,
+						DepMode: GoGetNoModules,
+						ModMode: ModUnset,
+					})
+					goModDirs = append(goModDirs, path)
+					break
+				}
+			}
+		}
+
 		return
 	}
-	if len(goModPaths) > 1 {
-		// currently not supported
-		baseDir = "."
-		commonRoot, nested := checkDirsNested(goModDirs)
-		if nested && commonRoot == "" {
-			useGoMod = true
-		} else {
-			useGoMod = false
-		}
-		if emitDiagnostics {
+
+	// If we are emitted diagnostics, report some details about the workspace structure.
+	if emitDiagnostics {
+		if totalModuleFiles > 1 {
+			_, nested := checkDirsNested(goModDirs)
+
 			if nested {
 				diagnostics.EmitMultipleGoModFoundNested(goModPaths)
 			} else {
 				diagnostics.EmitMultipleGoModFoundNotNested(goModPaths)
 			}
-		}
-		return
-	}
-	if emitDiagnostics {
-		if goModDirs[0] == "." {
-			diagnostics.EmitSingleRootGoModFound(goModPaths[0])
-		} else {
-			diagnostics.EmitSingleNonRootGoModFound(goModPaths[0])
+		} else if totalModuleFiles == 1 {
+			if goModDirs[0] == "." {
+				diagnostics.EmitSingleRootGoModFound(goModPaths[0])
+			} else {
+				diagnostics.EmitSingleNonRootGoModFound(goModPaths[0])
+			}
 		}
 	}
-	baseDir = goModDirs[0]
-	useGoMod = true
+
 	return
 }
 
-// DependencyInstallerMode is an enum describing how dependencies should be installed
-type DependencyInstallerMode int
+// Determines whether `str` starts with any of `prefixes`.
+func startsWithAnyOf(str string, prefixes []string) bool {
+	for _, prefix := range prefixes {
+		if relPath, err := filepath.Rel(str, prefix); err == nil && !strings.HasPrefix(relPath, "..") {
+			return true
+		}
+	}
+	return false
+}
 
-const (
-	// GoGetNoModules represents dependency installation using `go get` without modules
-	GoGetNoModules DependencyInstallerMode = iota
-	// GoGetWithModules represents dependency installation using `go get` with modules
-	GoGetWithModules
-	// Dep represent dependency installation using `dep ensure`
-	Dep
-	// Glide represents dependency installation using `glide install`
-	Glide
-)
-
-// Returns the appropriate DependencyInstallerMode for the current project
-func getDepMode(emitDiagnostics bool) (DependencyInstallerMode, string) {
-	bazelPaths := util.FindAllFilesWithName(".", "BUILD", "vendor")
-	bazelPaths = append(bazelPaths, util.FindAllFilesWithName(".", "BUILD.bazel", "vendor")...)
+// Finds Go workspaces in the current working directory.
+func GetWorkspaceInfo(emitDiagnostics bool) []GoWorkspace {
+	bazelPaths := slices.Concat(
+		util.FindAllFilesWithName(".", "BUILD", "vendor"),
+		util.FindAllFilesWithName(".", "BUILD.bazel", "vendor"),
+	)
 	if len(bazelPaths) > 0 {
 		// currently not supported
 		if emitDiagnostics {
@@ -115,36 +501,10 @@ func getDepMode(emitDiagnostics bool) (DependencyInstallerMode, string) {
 		}
 	}
 
-	goWorkPaths := util.FindAllFilesWithName(".", "go.work", "vendor")
-	if len(goWorkPaths) > 0 {
-		// currently not supported
-		if emitDiagnostics {
-			diagnostics.EmitGoWorkFound(goWorkPaths)
-		}
-	}
+	goWorkspaces, totalModuleFiles := getBuildRoots(emitDiagnostics)
+	log.Printf("Found %d go.mod file(s).\n", totalModuleFiles)
 
-	baseDir, useGoMod := findGoModFiles(emitDiagnostics)
-	if useGoMod {
-		log.Println("Found go.mod, enabling go modules")
-		return GoGetWithModules, baseDir
-	}
-
-	if util.FileExists("Gopkg.toml") {
-		if emitDiagnostics {
-			diagnostics.EmitGopkgTomlFound()
-		}
-		log.Println("Found Gopkg.toml, using dep instead of go get")
-		return Dep, "."
-	}
-
-	if util.FileExists("glide.yaml") {
-		if emitDiagnostics {
-			diagnostics.EmitGlideYamlFound()
-		}
-		log.Println("Found glide.yaml, using Glide instead of go get")
-		return Glide, "."
-	}
-	return GoGetNoModules, "."
+	return goWorkspaces
 }
 
 // ModMode corresponds to the possible values of the -mod flag for the Go compiler
@@ -194,38 +554,18 @@ func getModMode(depMode DependencyInstallerMode, baseDir string) ModMode {
 	return ModUnset
 }
 
-type BuildInfo struct {
-	DepMode DependencyInstallerMode
-	ModMode ModMode
-	BaseDir string
-}
-
-func GetBuildInfo(emitDiagnostics bool) BuildInfo {
-	depMode, baseDir := getDepMode(true)
-	modMode := getModMode(depMode, baseDir)
-	return BuildInfo{depMode, modMode, baseDir}
-}
-
-type GoVersionInfo struct {
-	// The version string, if any
-	Version string
-	// A value indicating whether a version string was found
-	Found bool
-}
-
 // Tries to open `go.mod` and read a go directive, returning the version and whether it was found.
-func TryReadGoDirective(buildInfo BuildInfo) GoVersionInfo {
-	if buildInfo.DepMode == GoGetWithModules {
-		versionRe := regexp.MustCompile(`(?m)^go[ \t\r]+([0-9]+\.[0-9]+(\.[0-9]+)?)$`)
-		goMod, err := os.ReadFile(filepath.Join(buildInfo.BaseDir, "go.mod"))
-		if err != nil {
-			log.Println("Failed to read go.mod to check for missing Go version")
-		} else {
-			matches := versionRe.FindSubmatch(goMod)
-			if matches != nil {
-				if len(matches) > 1 {
-					return GoVersionInfo{string(matches[1]), true}
-				}
+// The version string is returned in the "1.2.3" format.
+func tryReadGoDirective(path string) GoVersionInfo {
+	versionRe := regexp.MustCompile(`(?m)^go[ \t\r]+([0-9]+\.[0-9]+(\.[0-9]+)?)$`)
+	goMod, err := os.ReadFile(path)
+	if err != nil {
+		log.Println("Failed to read go.mod to check for missing Go version")
+	} else {
+		matches := versionRe.FindSubmatch(goMod)
+		if matches != nil {
+			if len(matches) > 1 {
+				return GoVersionInfo{string(matches[1]), true}
 			}
 		}
 	}

go/extractor/project/project_test.go

@@ -0,0 +1,27 @@
+package project
+
+import (
+	"path/filepath"
+	"testing"
+)
+
+func testStartsWithAnyOf(t *testing.T, path string, prefix string, expectation bool) {
+	result := startsWithAnyOf(path, []string{prefix})
+	if result != expectation {
+		t.Errorf("Expected startsWithAnyOf(%s, %s) to be %t, but it is %t.", path, prefix, expectation, result)
+	}
+}
+
+func TestStartsWithAnyOf(t *testing.T) {
+	testStartsWithAnyOf(t, ".", ".", true)
+	testStartsWithAnyOf(t, ".", "dir", true)
+	testStartsWithAnyOf(t, ".", filepath.Join("foo", "bar"), true)
+	testStartsWithAnyOf(t, "dir", "dir", true)
+	testStartsWithAnyOf(t, "foo", filepath.Join("foo", "bar"), true)
+	testStartsWithAnyOf(t, filepath.Join("foo", "bar"), filepath.Join("foo", "bar"), true)
+	testStartsWithAnyOf(t, filepath.Join("foo", "bar"), filepath.Join("foo", "bar", "baz"), true)
+
+	testStartsWithAnyOf(t, filepath.Join("foo", "bar"), "foo", false)
+	testStartsWithAnyOf(t, filepath.Join("foo", "bar"), "bar", false)
+	testStartsWithAnyOf(t, filepath.Join("foo", "bar"), filepath.Join("foo", "baz"), false)
+}

go/extractor/toolchain/toolchain.go

@@ -5,7 +5,11 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
+
+	"github.com/github/codeql-go/extractor/util"
+	"golang.org/x/mod/semver"
 )
 
 // Check if Go is installed in the environment.
@@ -36,6 +40,23 @@ func GetEnvGoVersion() string {
 	return goVersion
 }
 
+// Returns the current Go version in semver format, e.g. v1.14.4
+func GetEnvGoSemVer() string {
+	goVersion := GetEnvGoVersion()
+	if !strings.HasPrefix(goVersion, "go") {
+		log.Fatalf("Expected 'go version' output of the form 'go1.2.3'; got '%s'", goVersion)
+	}
+	// Go versions don't follow the SemVer format, but the only exception we normally care about
+	// is release candidates; so this is a horrible hack to convert e.g. `go1.22rc1` into `go1.22-rc1`
+	// which is compatible with the SemVer specification
+	rcIndex := strings.Index(goVersion, "rc")
+	if rcIndex != -1 {
+		return semver.Canonical("v"+goVersion[2:rcIndex]) + "-" + goVersion[rcIndex:]
+	} else {
+		return semver.Canonical("v" + goVersion[2:])
+	}
+}
+
 // The 'go version' command may output warnings on separate lines before
 // the actual version string is printed. This function parses the output
 // to retrieve just the version string.
@@ -47,3 +68,42 @@ func parseGoVersion(data string) string {
 	}
 	return strings.Fields(lastLine)[2]
 }
+
+// Returns a value indicating whether the system Go toolchain supports workspaces.
+func SupportsWorkspaces() bool {
+	return semver.Compare(GetEnvGoSemVer(), "v1.18.0") >= 0
+}
+
+// Run `go mod tidy -e` in the directory given by `path`.
+func TidyModule(path string) *exec.Cmd {
+	cmd := exec.Command("go", "mod", "tidy", "-e")
+	cmd.Dir = path
+	return cmd
+}
+
+// Run `go mod init` in the directory given by `path`.
+func InitModule(path string) *exec.Cmd {
+	moduleName := "codeql/auto-project"
+
+	if importpath := util.GetImportPath(); importpath != "" {
+		// This should be something like `github.com/user/repo`
+		moduleName = importpath
+
+		// If we are not initialising the new module in the root directory of the workspace,
+		// append the relative path to the module name.
+		if relPath, err := filepath.Rel(".", path); err != nil && relPath != "." {
+			moduleName = moduleName + "/" + relPath
+		}
+	}
+
+	modInit := exec.Command("go", "mod", "init", moduleName)
+	modInit.Dir = path
+	return modInit
+}
+
+// Constructs a command to run `go mod vendor -e` in the directory given by `path`.
+func VendorModule(path string) *exec.Cmd {
+	modVendor := exec.Command("go", "mod", "vendor", "-e")
+	modVendor.Dir = path
+	return modVendor
+}

go/extractor/util/util.go

@@ -6,10 +6,13 @@ import (
 	"io"
 	"io/fs"
 	"log"
+	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"runtime"
+	"slices"
 	"strings"
 )
 
@@ -204,13 +207,13 @@ func RunCmd(cmd *exec.Cmd) bool {
 	in, _ := cmd.StdinPipe()
 	err := cmd.Start()
 	if err != nil {
-		log.Printf("Running %s failed, continuing anyway: %s\n", cmd.Path, err.Error())
+		log.Printf("Running %s %v failed, continuing anyway: %s\n", cmd.Path, cmd.Args, err.Error())
 		return false
 	}
 	in.Close()
 	err = cmd.Wait()
 	if err != nil {
-		log.Printf("Running %s failed, continuing anyway: %s\n", cmd.Path, err.Error())
+		log.Printf("Running %s %v failed, continuing anyway: %s\n", cmd.Path, cmd.Args, err.Error())
 		return false
 	}
 
@@ -319,24 +322,90 @@ func FindAllFilesWithName(root string, name string, dirsToSkip ...string) []stri
 	return paths
 }
 
-func AnyGoFilesOutsideDirs(root string, dirsToSkip ...string) bool {
-	found := false
+// Returns an array of any Go source files in locations which do not have a `go.mod`
+// file in the same directory or higher up in the file hierarchy, relative to the `root`.
+func GoFilesOutsideDirs(root string, dirsToSkip ...string) []string {
+	result := []string{}
+
 	filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
-		if d.IsDir() {
-			for _, dirToSkip := range dirsToSkip {
-				if path == dirToSkip {
-					return filepath.SkipDir
-				}
-			}
+		if d.IsDir() && slices.Contains(dirsToSkip, path) {
+			return filepath.SkipDir
 		}
 		if filepath.Ext(d.Name()) == ".go" {
-			found = true
-			return filepath.SkipAll
+			log.Printf("Found stray Go source file in %s.\n", path)
+			result = append(result, path)
 		}
 		return nil
 	})
-	return found
+
+	return result
+}
+
+// For every file path in the input array, return the parent directory.
+func GetParentDirs(paths []string) []string {
+	dirs := make([]string, len(paths))
+	for i, path := range paths {
+		dirs[i] = filepath.Dir(path)
+	}
+	return dirs
+}
+
+// Returns the import path of the package being built, or "" if it cannot be determined.
+func GetImportPath() (importpath string) {
+	importpath = os.Getenv("LGTM_INDEX_IMPORT_PATH")
+	if importpath == "" {
+		repourl := os.Getenv("SEMMLE_REPO_URL")
+		if repourl == "" {
+			githubrepo := os.Getenv("GITHUB_REPOSITORY")
+			if githubrepo == "" {
+				log.Printf("Unable to determine import path, as neither LGTM_INDEX_IMPORT_PATH nor GITHUB_REPOSITORY is set\n")
+				return ""
+			} else {
+				importpath = "github.com/" + githubrepo
+			}
+		} else {
+			importpath = getImportPathFromRepoURL(repourl)
+			if importpath == "" {
+				log.Printf("Failed to determine import path from SEMMLE_REPO_URL '%s'\n", repourl)
+				return
+			}
+		}
+	}
+	log.Printf("Import path is '%s'\n", importpath)
+	return
+}
+
+// Returns the import path of the package being built from `repourl`, or "" if it cannot be
+// determined.
+func getImportPathFromRepoURL(repourl string) string {
+	// check for scp-like URL as in "git@github.com:github/codeql-go.git"
+	shorturl := regexp.MustCompile(`^([^@]+@)?([^:]+):([^/].*?)(\.git)?$`)
+	m := shorturl.FindStringSubmatch(repourl)
+	if m != nil {
+		return m[2] + "/" + m[3]
+	}
+
+	// otherwise parse as proper URL
+	u, err := url.Parse(repourl)
+	if err != nil {
+		log.Fatalf("Malformed repository URL '%s'\n", repourl)
+	}
+
+	if u.Scheme == "file" {
+		// we can't determine import paths from file paths
+		return ""
+	}
+
+	if u.Hostname() == "" || u.Path == "" {
+		return ""
+	}
+
+	host := u.Hostname()
+	path := u.Path
+	// strip off leading slashes and trailing `.git` if present
+	path = regexp.MustCompile(`^/+|\.git$`).ReplaceAllString(path, "")
+	return host + "/" + path
 }

go/extractor/util/util_test.go

@@ -1,4 +1,4 @@
-package main
+package util
 
 import "testing"
 

go/ql/consistency-queries/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.0.9
+
+No user-facing changes.
+
 ## 0.0.8
 
 No user-facing changes.

go/ql/consistency-queries/change-notes/released/0.0.9.md

@@ -0,0 +1,3 @@
+## 0.0.9
+
+No user-facing changes.

go/ql/consistency-queries/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.0.8
+lastReleaseVersion: 0.0.9

go/ql/consistency-queries/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql-go-consistency-queries
-version: 0.0.9-dev
+version: 0.0.9
 groups:
   - go
   - queries

go/ql/integration-tests/all-platforms/go/diagnostics/go-files-found-not-processed/test.expected

@@ -0,0 +1,4 @@
+extractedFiles
+| work/subdir/go.mod:0:0:0:0 | work/subdir/go.mod |
+| work/subdir/test.go:0:0:0:0 | work/subdir/test.go |
+#select

go/ql/integration-tests/all-platforms/go/diagnostics/go-files-found-not-processed/test.py

@@ -10,7 +10,7 @@ from diagnostics_test_utils import *
 goPath = os.path.join(os.path.abspath(os.getcwd()), ".go")
 os.environ['GOPATH'] = goPath
 os.environ['LGTM_INDEX_IMPORT_PATH'] = "test"
-run_codeql_database_create([], lang="go", source="work", db=None, runFunction=runUnsuccessfully)
+run_codeql_database_create([], lang="go", source="work")
 
 check_diagnostics()
 

go/ql/integration-tests/all-platforms/go/diagnostics/go-files-found-not-processed/test.ql

@@ -0,0 +1,8 @@
+import go
+import semmle.go.DiagnosticsReporting
+
+query predicate extractedFiles(File f) { any() }
+
+from string msg, int sev
+where reportableDiagnostics(_, msg, sev)
+select msg, sev

go/ql/integration-tests/all-platforms/go/diagnostics/unsupported-relative-path/test.py

@@ -10,7 +10,7 @@ from diagnostics_test_utils import *
 goPath = os.path.join(os.path.abspath(os.getcwd()), ".go")
 os.environ['GOPATH'] = goPath
 os.environ['GITHUB_REPOSITORY'] = "a/b"
-run_codeql_database_create([], lang="go", source="work", db=None, runFunction=runUnsuccessfully)
+run_codeql_database_create([], lang="go", source="work", db=None)
 
 check_diagnostics()
 

go/ql/integration-tests/all-platforms/go/go-mod-without-version/diagnostics.expected

@@ -0,0 +1,14 @@
+{
+  "markdownMessage": "A single `go.mod` file was found.\n\n`go.mod`",
+  "severity": "note",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/single-root-go-mod-found",
+    "name": "A single `go.mod` file was found in the root"
+  },
+  "visibility": {
+    "cliSummaryTable": false,
+    "statusPage": false,
+    "telemetry": true
+  }
+}

go/ql/integration-tests/all-platforms/go/go-mod-without-version/force_sequential_test_execution

@@ -0,0 +1,2 @@
+# go get has been observed to sometimes fail when multiple tests try to simultaneously fetch the same package.
+goget

go/ql/integration-tests/all-platforms/go/go-mod-without-version/src/go.mod

@@ -0,0 +1,3 @@
+require golang.org/x/net v0.0.0-20200505041828-1ed23360d12c
+
+module test

go/ql/integration-tests/all-platforms/go/go-mod-without-version/src/go.sum

@@ -0,0 +1,7 @@
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20200505041828-1ed23360d12c h1:zJ0mtu4jCalhKg6Oaukv6iIkb+cOvDrajDH9DH46Q4M=
+golang.org/x/net v0.0.0-20200505041828-1ed23360d12c/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

go/ql/integration-tests/all-platforms/go/go-mod-without-version/src/subdir/add.go

@@ -0,0 +1,5 @@
+package subdir
+
+func Add(a, b int) int {
+	return a + b
+}

go/ql/integration-tests/all-platforms/go/go-mod-without-version/src/test.go

@@ -0,0 +1,14 @@
+package test
+
+import (
+	"test/subdir"
+
+	"golang.org/x/net/ipv4"
+)
+
+func test() {
+
+	header := ipv4.Header{}
+	header.Version = subdir.Add(2, 2)
+
+}

go/ql/integration-tests/all-platforms/go/go-mod-without-version/test.expected

@@ -0,0 +1,5 @@
+extractedFiles
+| src/go.mod:0:0:0:0 | src/go.mod |
+| src/subdir/add.go:0:0:0:0 | src/subdir/add.go |
+| src/test.go:0:0:0:0 | src/test.go |
+#select

go/ql/integration-tests/all-platforms/go/go-mod-without-version/test.py

@@ -0,0 +1,18 @@
+import os
+import subprocess
+
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+# Set up a GOPATH relative to this test's root directory;
+# we set os.environ instead of using extra_env because we
+# need it to be set for the call to "go clean -modcache" later
+goPath = os.path.join(os.path.abspath(os.getcwd()), ".go")
+os.environ['GOPATH'] = goPath
+run_codeql_database_create([], lang="go", source="src")
+
+check_diagnostics()
+
+# Clean up the temporary GOPATH to prevent Bazel failures next
+# time the tests are run; see https://github.com/golang/go/issues/27161
+subprocess.call(["go", "clean", "-modcache"])

go/ql/integration-tests/all-platforms/go/go-mod-without-version/test.ql

@@ -0,0 +1,8 @@
+import go
+import semmle.go.DiagnosticsReporting
+
+query predicate extractedFiles(File f) { any() }
+
+from string msg, int sev
+where reportableDiagnostics(_, msg, sev)
+select msg, sev

go/ql/integration-tests/all-platforms/go/mixed-layout/diagnostics.expected

@@ -0,0 +1,28 @@
+{
+  "markdownMessage": "1 `go.work` file was found:\n\n`workspace/go.work`",
+  "severity": "note",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/go-work-found",
+    "name": "`go.work` file found"
+  },
+  "visibility": {
+    "cliSummaryTable": false,
+    "statusPage": false,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "Go files were found outside of the Go modules corresponding to these `go.mod` files.\n\n`workspace/subdir/go.mod`, `module/go.mod`",
+  "severity": "note",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/go-files-outside-go-modules",
+    "name": "Go files were found outside Go modules"
+  },
+  "visibility": {
+    "cliSummaryTable": false,
+    "statusPage": false,
+    "telemetry": true
+  }
+}

go/ql/integration-tests/all-platforms/go/mixed-layout/force_sequential_test_execution

@@ -0,0 +1,2 @@
+# go get has been observed to sometimes fail when multiple tests try to simultaneously fetch the same package.
+goget

go/ql/integration-tests/all-platforms/go/mixed-layout/src/module/go.mod

@@ -0,0 +1,5 @@
+go 1.14
+
+require golang.org/x/net v0.0.0-20200505041828-1ed23360d12c
+
+module module

go/ql/integration-tests/all-platforms/go/mixed-layout/src/module/go.sum

@@ -0,0 +1,7 @@
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20200505041828-1ed23360d12c h1:zJ0mtu4jCalhKg6Oaukv6iIkb+cOvDrajDH9DH46Q4M=
+golang.org/x/net v0.0.0-20200505041828-1ed23360d12c/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

go/ql/integration-tests/all-platforms/go/mixed-layout/src/module/test.go

@@ -0,0 +1,13 @@
+package subdir
+
+import (
+	"fmt"
+
+	"golang.org/x/net/ipv4"
+)
+
+func test() {
+
+	header := ipv4.Header{}
+	fmt.Print(header.String())
+}

go/ql/integration-tests/all-platforms/go/mixed-layout/src/stray-files/test.go

@@ -0,0 +1,13 @@
+package subdir
+
+import (
+	"fmt"
+
+	"golang.org/x/net/ipv4"
+)
+
+func test() {
+
+	header := ipv4.Header{}
+	fmt.Print(header.String())
+}

go/ql/integration-tests/all-platforms/go/mixed-layout/src/workspace/go.work

@@ -0,0 +1,3 @@
+go 1.22.0
+
+use ./subdir

go/ql/integration-tests/all-platforms/go/mixed-layout/src/workspace/subdir/go.mod

@@ -0,0 +1,5 @@
+go 1.22.0
+
+require golang.org/x/net v0.0.0-20200505041828-1ed23360d12c
+
+module subdir

go/ql/integration-tests/all-platforms/go/mixed-layout/src/workspace/subdir/go.sum

@@ -0,0 +1,7 @@
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20200505041828-1ed23360d12c h1:zJ0mtu4jCalhKg6Oaukv6iIkb+cOvDrajDH9DH46Q4M=
+golang.org/x/net v0.0.0-20200505041828-1ed23360d12c/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

go/ql/integration-tests/all-platforms/go/mixed-layout/src/workspace/subdir/test.go

@@ -0,0 +1,13 @@
+package subdir
+
+import (
+	"fmt"
+
+	"golang.org/x/net/ipv4"
+)
+
+func test() {
+
+	header := ipv4.Header{}
+	fmt.Print(header.String())
+}

go/ql/integration-tests/all-platforms/go/mixed-layout/test.expected

@@ -0,0 +1,8 @@
+extractedFiles
+| src/module/go.mod:0:0:0:0 | src/module/go.mod |
+| src/module/test.go:0:0:0:0 | src/module/test.go |
+| src/stray-files/go.mod:0:0:0:0 | src/stray-files/go.mod |
+| src/stray-files/test.go:0:0:0:0 | src/stray-files/test.go |
+| src/workspace/subdir/go.mod:0:0:0:0 | src/workspace/subdir/go.mod |
+| src/workspace/subdir/test.go:0:0:0:0 | src/workspace/subdir/test.go |
+#select

go/ql/integration-tests/all-platforms/go/mixed-layout/test.py

@@ -0,0 +1,18 @@
+import os
+import subprocess
+
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+# Set up a GOPATH relative to this test's root directory;
+# we set os.environ instead of using extra_env because we
+# need it to be set for the call to "go clean -modcache" later
+goPath = os.path.join(os.path.abspath(os.getcwd()), ".go")
+os.environ['GOPATH'] = goPath
+run_codeql_database_create([], lang="go", source="src")
+
+check_diagnostics()
+
+# Clean up the temporary GOPATH to prevent Bazel failures next
+# time the tests are run; see https://github.com/golang/go/issues/27161
+subprocess.call(["go", "clean", "-modcache"])

go/ql/integration-tests/all-platforms/go/mixed-layout/test.ql

@@ -0,0 +1,8 @@
+import go
+import semmle.go.DiagnosticsReporting
+
+query predicate extractedFiles(File f) { any() }
+
+from string msg, int sev
+where reportableDiagnostics(_, msg, sev)
+select msg, sev

go/ql/integration-tests/all-platforms/go/single-go-mod-and-go-files-not-under-it/diagnostics.expected

@@ -1,17 +1,3 @@
-{
-  "markdownMessage": "1 package could not be found:\n\n`subdir/subsubdir`.\n\nDefinitions in those packages may not be recognized by CodeQL, and files that use them may only be partially analyzed.\n\nCheck that the paths are correct and make sure any private packages can be accessed. If any of the packages are present in the repository then you may need a [custom build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
-  "severity": "warning",
-  "source": {
-    "extractorName": "go",
-    "id": "go/autobuilder/package-not-found",
-    "name": "Some packages could not be found"
-  },
-  "visibility": {
-    "cliSummaryTable": true,
-    "statusPage": true,
-    "telemetry": true
-  }
-}
 {
   "markdownMessage": "Go files were found outside of the Go modules corresponding to these `go.mod` files.\n\n`subdir/go.mod`",
   "severity": "note",

go/ql/integration-tests/all-platforms/go/single-go-mod-and-go-files-not-under-it/test.expected

@@ -1,2 +1,5 @@
-| Extraction failed in subdir/test.go with error cannot find package "subdir/subsubdir" in any of:\n\t(absolute path) (from $GOROOT)\n\t(absolute path) (from $GOPATH) | 2 |
-| Extraction failed in subdir/test.go with error could not import subdir/subsubdir (invalid package name: "") | 2 |
+extractedFiles
+| src/subdir/go.mod:0:0:0:0 | src/subdir/go.mod |
+| src/subdir/subsubdir/add.go:0:0:0:0 | src/subdir/subsubdir/add.go |
+| src/subdir/test.go:0:0:0:0 | src/subdir/test.go |
+#select

go/ql/integration-tests/all-platforms/go/single-go-mod-and-go-files-not-under-it/test.ql

@@ -1,6 +1,8 @@
 import go
 import semmle.go.DiagnosticsReporting
 
+query predicate extractedFiles(File f) { any() }
+
 from string msg, int sev
 where reportableDiagnostics(_, msg, sev)
 select msg, sev

go/ql/integration-tests/all-platforms/go/single-go-mod-in-root/test.expected

@@ -0,0 +1,5 @@
+extractedFiles
+| src/go.mod:0:0:0:0 | src/go.mod |
+| src/subdir/add.go:0:0:0:0 | src/subdir/add.go |
+| src/test.go:0:0:0:0 | src/test.go |
+#select

go/ql/integration-tests/all-platforms/go/single-go-mod-in-root/test.ql

@@ -1,6 +1,8 @@
 import go
 import semmle.go.DiagnosticsReporting
 
+query predicate extractedFiles(File f) { any() }
+
 from string msg, int sev
 where reportableDiagnostics(_, msg, sev)
 select msg, sev

go/ql/integration-tests/all-platforms/go/single-go-mod-not-in-root/test.expected

@@ -0,0 +1,5 @@
+extractedFiles
+| src/subdir/go.mod:0:0:0:0 | src/subdir/go.mod |
+| src/subdir/subsubdir/add.go:0:0:0:0 | src/subdir/subsubdir/add.go |
+| src/subdir/test.go:0:0:0:0 | src/subdir/test.go |
+#select

go/ql/integration-tests/all-platforms/go/single-go-mod-not-in-root/test.ql

@@ -1,6 +1,8 @@
 import go
 import semmle.go.DiagnosticsReporting
 
+query predicate extractedFiles(File f) { any() }
+
 from string msg, int sev
 where reportableDiagnostics(_, msg, sev)
 select msg, sev

go/ql/integration-tests/all-platforms/go/single-go-work-not-in-root/diagnostics.expected

@@ -26,17 +26,3 @@
     "telemetry": true
   }
 }
-{
-  "markdownMessage": "2 packages could not be found:\n\n`subdir1/subsubdir1`, `subdir2/subsubdir2`.\n\nDefinitions in those packages may not be recognized by CodeQL, and files that use them may only be partially analyzed.\n\nCheck that the paths are correct and make sure any private packages can be accessed. If any of the packages are present in the repository then you may need a [custom build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
-  "severity": "warning",
-  "source": {
-    "extractorName": "go",
-    "id": "go/autobuilder/package-not-found",
-    "name": "Some packages could not be found"
-  },
-  "visibility": {
-    "cliSummaryTable": true,
-    "statusPage": true,
-    "telemetry": true
-  }
-}

go/ql/integration-tests/all-platforms/go/single-go-work-not-in-root/test.expected

@@ -1,4 +1,8 @@
-| Extraction failed in modules/subdir1/test.go with error cannot find package "subdir1/subsubdir1" in any of:\n\t(absolute path) (from $GOROOT)\n\t(absolute path) (from $GOPATH) | 2 |
-| Extraction failed in modules/subdir1/test.go with error could not import subdir1/subsubdir1 (invalid package name: "") | 2 |
-| Extraction failed in modules/subdir2/test.go with error cannot find package "subdir2/subsubdir2" in any of:\n\t(absolute path) (from $GOROOT)\n\t(absolute path) (from $GOPATH) | 2 |
-| Extraction failed in modules/subdir2/test.go with error could not import subdir2/subsubdir2 (invalid package name: "") | 2 |
+extractedFiles
+| src/modules/subdir1/go.mod:0:0:0:0 | src/modules/subdir1/go.mod |
+| src/modules/subdir1/subsubdir1/add.go:0:0:0:0 | src/modules/subdir1/subsubdir1/add.go |
+| src/modules/subdir1/test.go:0:0:0:0 | src/modules/subdir1/test.go |
+| src/modules/subdir2/go.mod:0:0:0:0 | src/modules/subdir2/go.mod |
+| src/modules/subdir2/subsubdir2/add.go:0:0:0:0 | src/modules/subdir2/subsubdir2/add.go |
+| src/modules/subdir2/test.go:0:0:0:0 | src/modules/subdir2/test.go |
+#select

go/ql/integration-tests/all-platforms/go/single-go-work-not-in-root/test.ql

@@ -1,6 +1,8 @@
 import go
 import semmle.go.DiagnosticsReporting
 
+query predicate extractedFiles(File f) { any() }
+
 from string msg, int sev
 where reportableDiagnostics(_, msg, sev)
 select msg, sev

go/ql/integration-tests/all-platforms/go/two-go-mods-nested-none-in-root/diagnostics.expected

@@ -12,17 +12,3 @@
     "telemetry": true
   }
 }
-{
-  "markdownMessage": "2 packages could not be found:\n\n`test/subdir2`, `subdir1/subsubdir1`.\n\nDefinitions in those packages may not be recognized by CodeQL, and files that use them may only be partially analyzed.\n\nCheck that the paths are correct and make sure any private packages can be accessed. If any of the packages are present in the repository then you may need a [custom build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
-  "severity": "warning",
-  "source": {
-    "extractorName": "go",
-    "id": "go/autobuilder/package-not-found",
-    "name": "Some packages could not be found"
-  },
-  "visibility": {
-    "cliSummaryTable": true,
-    "statusPage": true,
-    "telemetry": true
-  }
-}

go/ql/integration-tests/all-platforms/go/two-go-mods-nested-none-in-root/test.expected

@@ -1,4 +1,8 @@
-| Extraction failed in subdir0/subdir1/test.go with error cannot find package "subdir1/subsubdir1" in any of:\n\t(absolute path) (from $GOROOT)\n\t(absolute path) (from $GOPATH) | 2 |
-| Extraction failed in subdir0/subdir1/test.go with error could not import subdir1/subsubdir1 (invalid package name: "") | 2 |
-| Extraction failed in subdir0/test.go with error cannot find package "test/subdir2" in any of:\n\t(absolute path) (from $GOROOT)\n\t(absolute path) (from $GOPATH) | 2 |
-| Extraction failed in subdir0/test.go with error could not import test/subdir2 (invalid package name: "") | 2 |
+extractedFiles
+| src/subdir0/go.mod:0:0:0:0 | src/subdir0/go.mod |
+| src/subdir0/subdir1/go.mod:0:0:0:0 | src/subdir0/subdir1/go.mod |
+| src/subdir0/subdir1/subsubdir1/add.go:0:0:0:0 | src/subdir0/subdir1/subsubdir1/add.go |
+| src/subdir0/subdir1/test.go:0:0:0:0 | src/subdir0/subdir1/test.go |
+| src/subdir0/subdir2/add.go:0:0:0:0 | src/subdir0/subdir2/add.go |
+| src/subdir0/test.go:0:0:0:0 | src/subdir0/test.go |
+#select

go/ql/integration-tests/all-platforms/go/two-go-mods-nested-none-in-root/test.ql

@@ -1,6 +1,8 @@
 import go
 import semmle.go.DiagnosticsReporting
 
+query predicate extractedFiles(File f) { any() }
+
 from string msg, int sev
 where reportableDiagnostics(_, msg, sev)
 select msg, sev

go/ql/integration-tests/all-platforms/go/two-go-mods-nested-one-in-root/test.expected

@@ -0,0 +1,8 @@
+extractedFiles
+| src/go.mod:0:0:0:0 | src/go.mod |
+| src/subdir1/go.mod:0:0:0:0 | src/subdir1/go.mod |
+| src/subdir1/subsubdir1/add.go:0:0:0:0 | src/subdir1/subsubdir1/add.go |
+| src/subdir1/test.go:0:0:0:0 | src/subdir1/test.go |
+| src/subdir2/add.go:0:0:0:0 | src/subdir2/add.go |
+| src/test.go:0:0:0:0 | src/test.go |
+#select

go/ql/integration-tests/all-platforms/go/two-go-mods-nested-one-in-root/test.ql

@@ -1,6 +1,8 @@
 import go
 import semmle.go.DiagnosticsReporting
 
+query predicate extractedFiles(File f) { any() }
+
 from string msg, int sev
 where reportableDiagnostics(_, msg, sev)
 select msg, sev

go/ql/integration-tests/all-platforms/go/two-go-mods-not-nested/diagnostics.expected

@@ -12,17 +12,3 @@
     "telemetry": true
   }
 }
-{
-  "markdownMessage": "2 packages could not be found:\n\n`subdir1/subsubdir1`, `subdir2/subsubdir2`.\n\nDefinitions in those packages may not be recognized by CodeQL, and files that use them may only be partially analyzed.\n\nCheck that the paths are correct and make sure any private packages can be accessed. If any of the packages are present in the repository then you may need a [custom build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
-  "severity": "warning",
-  "source": {
-    "extractorName": "go",
-    "id": "go/autobuilder/package-not-found",
-    "name": "Some packages could not be found"
-  },
-  "visibility": {
-    "cliSummaryTable": true,
-    "statusPage": true,
-    "telemetry": true
-  }
-}

go/ql/integration-tests/all-platforms/go/two-go-mods-not-nested/test.expected

@@ -1,4 +1,8 @@
-| Extraction failed in subdir1/test.go with error cannot find package "subdir1/subsubdir1" in any of:\n\t(absolute path) (from $GOROOT)\n\t(absolute path) (from $GOPATH) | 2 |
-| Extraction failed in subdir1/test.go with error could not import subdir1/subsubdir1 (invalid package name: "") | 2 |
-| Extraction failed in subdir2/test.go with error cannot find package "subdir2/subsubdir2" in any of:\n\t(absolute path) (from $GOROOT)\n\t(absolute path) (from $GOPATH) | 2 |
-| Extraction failed in subdir2/test.go with error could not import subdir2/subsubdir2 (invalid package name: "") | 2 |
+extractedFiles
+| src/subdir1/go.mod:0:0:0:0 | src/subdir1/go.mod |
+| src/subdir1/subsubdir1/add.go:0:0:0:0 | src/subdir1/subsubdir1/add.go |
+| src/subdir1/test.go:0:0:0:0 | src/subdir1/test.go |
+| src/subdir2/go.mod:0:0:0:0 | src/subdir2/go.mod |
+| src/subdir2/subsubdir2/add.go:0:0:0:0 | src/subdir2/subsubdir2/add.go |
+| src/subdir2/test.go:0:0:0:0 | src/subdir2/test.go |
+#select

go/ql/integration-tests/all-platforms/go/two-go-mods-not-nested/test.ql

@@ -1,6 +1,8 @@
 import go
 import semmle.go.DiagnosticsReporting
 
+query predicate extractedFiles(File f) { any() }
+
 from string msg, int sev
 where reportableDiagnostics(_, msg, sev)
 select msg, sev

go/ql/integration-tests/all-platforms/go/two-go-mods-one-failure/diagnostics.expected

@@ -0,0 +1,28 @@
+{
+  "markdownMessage": "2 `go.mod` files were found:\n\n`subdir1/go.mod`, `subdir2/go.mod`",
+  "severity": "note",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/multiple-go-mod-found-not-nested",
+    "name": "Multiple `go.mod` files found, not all nested under one root `go.mod` file"
+  },
+  "visibility": {
+    "cliSummaryTable": false,
+    "statusPage": false,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "The following 1 Go project could not be extracted successfully:\n\n`subdir2`\n",
+  "severity": "warning",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/extraction-failed-for-project",
+    "name": "Unable to extract 1 Go projects"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}

go/ql/integration-tests/all-platforms/go/two-go-mods-one-failure/force_sequential_test_execution

@@ -0,0 +1,2 @@
+# go get has been observed to sometimes fail when multiple tests try to simultaneously fetch the same package.
+goget

go/ql/integration-tests/all-platforms/go/two-go-mods-one-failure/src/subdir1/go.mod

@@ -0,0 +1,5 @@
+go 1.14
+
+require golang.org/x/net v0.0.0-20200505041828-1ed23360d12c
+
+module subdir1

go/ql/integration-tests/all-platforms/go/two-go-mods-one-failure/src/subdir1/go.sum

@@ -0,0 +1,7 @@
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20200505041828-1ed23360d12c h1:zJ0mtu4jCalhKg6Oaukv6iIkb+cOvDrajDH9DH46Q4M=
+golang.org/x/net v0.0.0-20200505041828-1ed23360d12c/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=