XXX: Disable dynamic join in some cases

Java: include link to `remote source` in TrustBoundaryViolation.ql

.bazelrc

@@ -11,7 +11,8 @@ common --override_module=semmle_code=%workspace%/misc/bazel/semmle_code_stub
 build --repo_env=CC=clang --repo_env=CXX=clang++
 
 build:linux --cxxopt=-std=c++20
-build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64
+# we currently cannot built the swift extractor for ARM
+build:macos --cxxopt=-std=c++20 --copt=-arch --copt=x86_64 --linkopt=-arch --linkopt=x86_64
 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor
 
 # this requires developer mode, but is required to have pack installer functioning

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 1.0.0-dev
+version: 0.13.2-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/SSAConstruction.qll

@@ -317,6 +317,7 @@ private module Cached {
   }
 
   cached
+  pragma[no_dynamic_join_order]
   Instruction getChiInstructionTotalOperand(ChiInstruction chiInstr) {
     exists(
       Alias::VirtualVariable vvar, OldInstruction oldInstr, OldBlock defBlock, int defRank,

cpp/ql/lib/semmle/code/cpp/ir/implementation/unaliased_ssa/internal/SSAConstruction.qll

@@ -317,6 +317,7 @@ private module Cached {
   }
 
   cached
+  pragma[no_dynamic_join_order]
   Instruction getChiInstructionTotalOperand(ChiInstruction chiInstr) {
     exists(
       Alias::VirtualVariable vvar, OldInstruction oldInstr, OldBlock defBlock, int defRank,

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 1.0.0-dev
+version: 0.9.13-dev
 groups:
   - cpp
   - queries

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

@@ -544,51 +544,6 @@ namespace Semmle.Autobuild.CSharp.Tests
             Assert.Equal(2, vcvarsfiles.Length);
         }
 
-        [Fact]
-        public void TestLinuxBuildlessExtractionSuccess()
-        {
-            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
-            actions.FileExists["csharp.log"] = true;
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
-            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
-            actions.EnumerateDirectories[@"C:\Project"] = "";
-
-            var autobuilder = CreateAutoBuilder(false, buildless: "true");
-            TestAutobuilderScript(autobuilder, 0, 1);
-        }
-
-        [Fact]
-        public void TestLinuxBuildlessExtractionFailed()
-        {
-            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 10;
-            actions.FileExists["csharp.log"] = true;
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
-            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
-            actions.EnumerateDirectories[@"C:\Project"] = "";
-
-            var autobuilder = CreateAutoBuilder(false, buildless: "true");
-            TestAutobuilderScript(autobuilder, 10, 1);
-        }
-
-        [Fact]
-        public void TestLinuxBuildlessExtractionSolution()
-        {
-            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
-            actions.FileExists["csharp.log"] = true;
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
-            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
-            actions.EnumerateDirectories[@"C:\Project"] = "";
-
-            var autobuilder = CreateAutoBuilder(false, buildless: "true");
-            TestAutobuilderScript(autobuilder, 0, 1);
-        }
-
         private void TestAutobuilderScript(CSharpAutobuilder autobuilder, int expectedOutput, int commandsRun)
         {
             Assert.Equal(expectedOutput, autobuilder.GetBuildScript().Run(actions, StartCallback, EndCallback));
@@ -677,21 +632,6 @@ namespace Semmle.Autobuild.CSharp.Tests
             TestAutobuilderScript(autobuilder, 0, 1);
         }
 
-        [Fact]
-        public void TestSkipNugetBuildless()
-        {
-            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
-            actions.FileExists["csharp.log"] = true;
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
-            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
-            actions.EnumerateDirectories[@"C:\Project"] = "";
-
-            var autobuilder = CreateAutoBuilder(false, buildless: "true");
-            TestAutobuilderScript(autobuilder, 0, 1);
-        }
-
         [Fact]
         public void TestDotnetVersionNotInstalled()
         {

csharp/autobuilder/Semmle.Autobuild.CSharp/Semmle.Autobuild.CSharp.csproj

@@ -5,6 +5,7 @@
   <ItemGroup>
     <ProjectReference Include="..\..\extractor\Semmle.Util\Semmle.Util.csproj" />
     <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj" />
+    <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.Standalone\Semmle.Extraction.CSharp.Standalone.csproj" />
     <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.DependencyFetching\Semmle.Extraction.CSharp.DependencyFetching.csproj" />
     <ProjectReference Include="..\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
   </ItemGroup>

csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs

@@ -10,17 +10,7 @@ namespace Semmle.Autobuild.CSharp
     {
         public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
         {
-            if (builder.CodeQLExtractorLangRoot is null
-                || builder.CodeQlPlatform is null)
-            {
-                return BuildScript.Failure;
-            }
-
-            var standalone = builder.Actions.PathCombine(builder.CodeQLExtractorLangRoot, "tools", builder.CodeQlPlatform, "Semmle.Extraction.CSharp.Standalone");
-            var cmd = new CommandBuilder(builder.Actions);
-            cmd.RunCommand(standalone);
-
-            return cmd.Script;
+            return BuildScript.Create(_ => Semmle.Extraction.CSharp.Standalone.Program.Main([]));
         }
     }
 }

csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs

@@ -73,16 +73,6 @@ namespace Semmle.Autobuild.Shared
         /// A logger.
         /// </summary>
         ILogger Logger { get; }
-
-        /// <summary>
-        /// Value of CODEQL_EXTRACTOR_<LANG>_ROOT environment variable.
-        /// </summary>
-        string? CodeQLExtractorLangRoot { get; }
-
-        /// <summary>
-        /// Value of CODEQL_PLATFORM environment variable.
-        /// </summary>
-        string? CodeQlPlatform { get; }
     }
 
     /// <summary>
@@ -197,9 +187,6 @@ namespace Semmle.Autobuild.Shared
                 return ret ?? new List<IProjectOrSolution>();
             });
 
-            CodeQLExtractorLangRoot = Actions.GetEnvironmentVariable(EnvVars.Root(this.Options.Language));
-            CodeQlPlatform = Actions.GetEnvironmentVariable(EnvVars.Platform);
-
             TrapDir = RequireEnvironmentVariable(EnvVars.TrapDir(this.Options.Language));
             SourceArchiveDir = RequireEnvironmentVariable(EnvVars.SourceArchiveDir(this.Options.Language));
             DiagnosticsDir = RequireEnvironmentVariable(EnvVars.DiagnosticDir(this.Options.Language));
@@ -364,15 +351,5 @@ namespace Semmle.Autobuild.Shared
                 diagnostics.Dispose();
             }
         }
-
-        /// <summary>
-        /// Value of CODEQL_EXTRACTOR_<LANG>_ROOT environment variable.
-        /// </summary>
-        public string? CodeQLExtractorLangRoot { get; }
-
-        /// <summary>
-        /// Value of CODEQL_PLATFORM environment variable.
-        /// </summary>
-        public string? CodeQlPlatform { get; }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

@@ -60,6 +60,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         /// </summary>
         public const string FallbackNugetFeeds = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK";
 
+        /// <summary>
+        /// Controls whether to include NuGet feeds from nuget.config files in the fallback restore logic.
+        /// </summary>
+        public const string AddNugetConfigFeedsToFallback = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK_INCLUDE_NUGET_CONFIG_FEEDS";
+
         /// <summary>
         /// Specifies the path to the nuget executable to be used for package restoration.
         /// </summary>

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -98,12 +98,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             logger.LogInfo($"Checking NuGet feed responsiveness: {checkNugetFeedResponsiveness}");
             compilationInfoContainer.CompilationInfos.Add(("NuGet feed responsiveness checked", checkNugetFeedResponsiveness ? "1" : "0"));
 
+            HashSet<string>? explicitFeeds = null;
+
             try
             {
-                if (checkNugetFeedResponsiveness && !CheckFeeds())
+                if (checkNugetFeedResponsiveness && !CheckFeeds(out explicitFeeds))
                 {
                     // todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too.
-                    var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds();
+                    var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds(explicitFeeds);
                     return unresponsiveMissingPackageLocation is null
                         ? []
                         : [unresponsiveMissingPackageLocation];
@@ -163,7 +165,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             LogAllUnusedPackages(dependencies);
 
             var missingPackageLocation = checkNugetFeedResponsiveness
-                ? DownloadMissingPackagesFromSpecificFeeds()
+                ? DownloadMissingPackagesFromSpecificFeeds(explicitFeeds)
                 : DownloadMissingPackages();
 
             if (missingPackageLocation is not null)
@@ -173,13 +175,24 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return assemblyLookupLocations;
         }
 
-        private List<string> GetReachableFallbackNugetFeeds()
+        private List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNugetConfigs)
         {
             var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet();
             if (fallbackFeeds.Count == 0)
             {
                 fallbackFeeds.Add(PublicNugetOrgFeed);
-                logger.LogInfo($"No fallback Nuget feeds specified. Using default feed: {PublicNugetOrgFeed}");
+                logger.LogInfo($"No fallback Nuget feeds specified. Adding default feed: {PublicNugetOrgFeed}");
+
+                var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback);
+                logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}");
+
+                if (shouldAddNugetConfigFeeds && feedsFromNugetConfigs?.Count > 0)
+                {
+                    // There are some feeds in `feedsFromNugetConfigs` that have already been checked for reachability, we could skip those.
+                    // But we might use different responsiveness testing settings when we try them in the fallback logic, so checking them again is safer.
+                    fallbackFeeds.UnionWith(feedsFromNugetConfigs);
+                    logger.LogInfo($"Using Nuget feeds from nuget.config files as fallback feeds: {string.Join(", ", feedsFromNugetConfigs.OrderBy(f => f))}");
+                }
             }
 
             logger.LogInfo($"Checking fallback Nuget feed reachability on feeds: {string.Join(", ", fallbackFeeds.OrderBy(f => f))}");
@@ -194,6 +207,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 logger.LogInfo($"Reachable fallback Nuget feeds: {string.Join(", ", reachableFallbackFeeds.OrderBy(f => f))}");
             }
 
+            compilationInfoContainer.CompilationInfos.Add(("Reachable fallback Nuget feed count", reachableFallbackFeeds.Count.ToString()));
+
             return reachableFallbackFeeds;
         }
 
@@ -272,9 +287,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             compilationInfoContainer.CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
         }
 
-        private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds()
+        private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds(HashSet<string>? feedsFromNugetConfigs)
         {
-            var reachableFallbackFeeds = GetReachableFallbackNugetFeeds();
+            var reachableFallbackFeeds = GetReachableFallbackNugetFeeds(feedsFromNugetConfigs);
             if (reachableFallbackFeeds.Count > 0)
             {
                 return DownloadMissingPackages(fallbackNugetFeeds: reachableFallbackFeeds);
@@ -623,10 +638,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return (timeoutMilliSeconds, tryCount);
         }
 
-        private bool CheckFeeds()
+        private bool CheckFeeds(out HashSet<string> explicitFeeds)
         {
             logger.LogInfo("Checking Nuget feeds...");
-            var (explicitFeeds, allFeeds) = GetAllFeeds();
+            (explicitFeeds, var allFeeds) = GetAllFeeds();
 
             var excludedFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck)
                 .ToHashSet() ?? [];

csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs

@@ -11,26 +11,20 @@ namespace Semmle.Extraction.CSharp.Entities
     {
         internal readonly ConcurrentDictionary<string, int> messageCounts = new();
 
-        private static (string Cwd, string[] Args) settings;
-        private static int hashCode;
-
-        public static (string Cwd, string[] Args) Settings
-        {
-            get { return settings; }
-            set
-            {
-                settings = value;
-                hashCode = settings.Cwd.GetHashCode();
-                for (var i = 0; i < settings.Args.Length; i++)
-                {
-                    hashCode = HashCode.Combine(hashCode, settings.Args[i].GetHashCode());
-                }
-            }
-        }
+        private readonly string cwd;
+        private readonly string[] args;
+        private readonly int hashCode;
 
 #nullable disable warnings
         private Compilation(Context cx) : base(cx, null)
         {
+            cwd = cx.Extractor.Cwd;
+            args = cx.Extractor.Args;
+            hashCode = cwd.GetHashCode();
+            for (var i = 0; i < args.Length; i++)
+            {
+                hashCode = HashCode.Combine(hashCode, args[i].GetHashCode());
+            }
         }
 #nullable restore warnings
 
@@ -38,14 +32,14 @@ namespace Semmle.Extraction.CSharp.Entities
         {
             var assembly = Assembly.CreateOutputAssembly(Context);
 
-            trapFile.compilations(this, FileUtils.ConvertToUnix(Compilation.Settings.Cwd));
+            trapFile.compilations(this, FileUtils.ConvertToUnix(cwd));
             trapFile.compilation_assembly(this, assembly);
 
             // Arguments
             var expandedIndex = 0;
-            for (var i = 0; i < Compilation.Settings.Args.Length; i++)
+            for (var i = 0; i < args.Length; i++)
             {
-                var arg = Compilation.Settings.Args[i];
+                var arg = args[i];
                 trapFile.compilation_args(this, i, arg);
 
                 if (CommandLineExtensions.IsFileArgument(arg))

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs

@@ -97,7 +97,8 @@ namespace Semmle.Extraction.CSharp
             stopwatch.Start();
 
             var options = Options.CreateWithEnvironment(args);
-            Entities.Compilation.Settings = (Directory.GetCurrentDirectory(), options.CompilerArguments.ToArray());
+            var workingDirectory = Directory.GetCurrentDirectory();
+            var compilerArgs = options.CompilerArguments.ToArray();
 
             using var logger = MakeLogger(options.Verbosity, options.Console);
 
@@ -123,7 +124,7 @@ namespace Semmle.Extraction.CSharp
 
                 var compilerArguments = CSharpCommandLineParser.Default.Parse(
                     compilerVersion.ArgsWithResponse,
-                    Entities.Compilation.Settings.Cwd,
+                    workingDirectory,
                     compilerVersion.FrameworkPath,
                     compilerVersion.AdditionalReferenceDirectories
                     );
@@ -131,7 +132,7 @@ namespace Semmle.Extraction.CSharp
                 if (compilerArguments is null)
                 {
                     var sb = new StringBuilder();
-                    sb.Append("  Failed to parse command line: ").AppendList(" ", Entities.Compilation.Settings.Args);
+                    sb.Append("  Failed to parse command line: ").AppendList(" ", compilerArgs);
                     logger.Log(Severity.Error, sb.ToString());
                     ++analyser.CompilationErrors;
                     return ExitCode.Failed;
@@ -143,7 +144,7 @@ namespace Semmle.Extraction.CSharp
                     return ExitCode.Ok;
                 }
 
-                return AnalyseTracing(analyser, compilerArguments, options, canonicalPathCache, stopwatch);
+                return AnalyseTracing(workingDirectory, compilerArgs, analyser, compilerArguments, options, canonicalPathCache, stopwatch);
             }
             catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
             {
@@ -376,6 +377,8 @@ namespace Semmle.Extraction.CSharp
         }
 
         private static ExitCode AnalyseTracing(
+            string cwd,
+            string[] args,
             TracingAnalyser analyser,
             CSharpCommandLineArguments compilerArguments,
             Options options,
@@ -420,7 +423,7 @@ namespace Semmle.Extraction.CSharp
                             .WithMetadataImportOptions(MetadataImportOptions.All)
                         );
                 },
-                (compilation, options) => analyser.EndInitialize(compilerArguments, options, compilation),
+                (compilation, options) => analyser.EndInitialize(compilerArguments, options, compilation, cwd, args),
                 () => { });
         }
 

csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs

@@ -16,12 +16,10 @@ namespace Semmle.Extraction.CSharp
         public void Initialize(string outputPath, IEnumerable<(string, string)> compilationInfos, CSharpCompilation compilationIn, CommonOptions options)
         {
             compilation = compilationIn;
-            extractor = new StandaloneExtractor(outputPath, compilationInfos, Logger, PathTransformer, options);
+            extractor = new StandaloneExtractor(Directory.GetCurrentDirectory(), outputPath, compilationInfos, Logger, PathTransformer, options);
             this.options = options;
             LogExtractorInfo(Extraction.Extractor.Version);
             SetReferencePaths();
-
-            Entities.Compilation.Settings = (Directory.GetCurrentDirectory(), Array.Empty<string>());
         }
 
 #nullable disable warnings

csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs

@@ -38,13 +38,15 @@ namespace Semmle.Extraction.CSharp
         public void EndInitialize(
            CSharpCommandLineArguments commandLineArguments,
            CommonOptions options,
-           CSharpCompilation compilation)
+           CSharpCompilation compilation,
+           string cwd,
+           string[] args)
         {
             if (!init)
                 throw new InternalError("EndInitialize called without BeginInitialize returning true");
             this.options = options;
             this.compilation = compilation;
-            this.extractor = new TracingExtractor(GetOutputName(compilation, commandLineArguments), Logger, PathTransformer, options);
+            this.extractor = new TracingExtractor(cwd, args, GetOutputName(compilation, commandLineArguments), Logger, PathTransformer, options);
             LogDiagnostics();
 
             SetReferencePaths();

csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs

@@ -10,6 +10,8 @@ namespace Semmle.Extraction
     /// </summary>
     public abstract class Extractor
     {
+        public string Cwd { get; init; }
+        public string[] Args { get; init; }
         public abstract ExtractorMode Mode { get; }
         public string OutputPath { get; }
         public IEnumerable<CompilationInfo> CompilationInfos { get; }
@@ -19,12 +21,14 @@ namespace Semmle.Extraction
         /// </summary>
         /// <param name="logger">The object used for logging.</param>
         /// <param name="pathTransformer">The object used for path transformations.</param>
-        protected Extractor(string outputPath, IEnumerable<CompilationInfo> compilationInfos, ILogger logger, PathTransformer pathTransformer)
+        protected Extractor(string cwd, string[] args, string outputPath, IEnumerable<CompilationInfo> compilationInfos, ILogger logger, PathTransformer pathTransformer)
         {
             OutputPath = outputPath;
             Logger = logger;
             PathTransformer = pathTransformer;
             CompilationInfos = compilationInfos;
+            Cwd = cwd;
+            Args = args;
         }
 
         // Limit the number of error messages in the log file

csharp/extractor/Semmle.Extraction/Extractor/StandaloneExtractor.cs

@@ -12,7 +12,8 @@ namespace Semmle.Extraction
         /// </summary>
         /// <param name="logger">The object used for logging.</param>
         /// <param name="pathTransformer">The object used for path transformations.</param>
-        public StandaloneExtractor(string outputPath, IEnumerable<(string, string)> compilationInfos, ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(outputPath, compilationInfos, logger, pathTransformer)
+        public StandaloneExtractor(string cwd, string outputPath, IEnumerable<(string, string)> compilationInfos, ILogger logger, PathTransformer pathTransformer, CommonOptions options)
+            : base(cwd, [], outputPath, compilationInfos, logger, pathTransformer)
         {
             Mode = ExtractorMode.Standalone;
             if (options.QlTest)

csharp/extractor/Semmle.Extraction/Extractor/TracingExtractor.cs

@@ -1,4 +1,3 @@
-using System.Linq;
 using Semmle.Util.Logging;
 
 namespace Semmle.Extraction
@@ -13,7 +12,8 @@ namespace Semmle.Extraction
         /// <param name="outputPath">The name of the output DLL/EXE, or null if not specified (standalone extraction).</param>
         /// <param name="logger">The object used for logging.</param>
         /// <param name="pathTransformer">The object used for path transformations.</param>
-        public TracingExtractor(string outputPath, ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(outputPath, Enumerable.Empty<(string, string)>(), logger, pathTransformer)
+        public TracingExtractor(string cwd, string[] args, string outputPath, ILogger logger, PathTransformer pathTransformer, CommonOptions options)
+            : base(cwd, args, outputPath, [], logger, pathTransformer)
         {
             Mode = ExtractorMode.None;
             if (options.QlTest)

csharp/ql/integration-tests/all-platforms/standalone_resx/CompilationInfo.expected

@@ -3,6 +3,7 @@
 | Failed solution restore with package source error | 0.0 |
 | NuGet feed responsiveness checked | 1.0 |
 | Project files on filesystem | 1.0 |
+| Reachable fallback Nuget feed count | 1.0 |
 | Resource extraction enabled | 1.0 |
 | Restored .NET framework variants | 1.0 |
 | Restored projects through solution files | 0.0 |

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/CompilationInfo.expected

@@ -4,6 +4,7 @@
 | Fallback nuget restore | 1.0 |
 | NuGet feed responsiveness checked | 1.0 |
 | Project files on filesystem | 1.0 |
+| Reachable fallback Nuget feed count | 1.0 |
 | Resolved assembly conflicts | 7.0 |
 | Resource extraction enabled | 0.0 |
 | Restored .NET framework variants | 0.0 |

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.expected

@@ -3,6 +3,7 @@
 | Inherited Nuget feed count | 1.0 |
 | NuGet feed responsiveness checked | 1.0 |
 | Project files on filesystem | 1.0 |
+| Reachable fallback Nuget feed count | 1.0 |
 | Resolved assembly conflicts | 7.0 |
 | Resource extraction enabled | 0.0 |
 | Restored .NET framework variants | 0.0 |

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/Assemblies.expected

@@ -0,0 +1 @@
+| [...]/newtonsoft.json/13.0.3/lib/net6.0/Newtonsoft.Json.dll |

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/Assemblies.ql

@@ -0,0 +1,11 @@
+import csharp
+
+private string getPath(Assembly a) {
+  not a.getCompilation().getOutputAssembly() = a and
+  exists(string s | s = a.getFile().getAbsolutePath() |
+    result = "[...]/" + s.substring(s.indexOf("newtonsoft.json"), s.length())
+  )
+}
+
+from Assembly a
+select getPath(a)

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/CompilationInfo.expected

@@ -0,0 +1,16 @@
+| All Nuget feeds reachable | 0.0 |
+| Fallback nuget restore | 1.0 |
+| NuGet feed responsiveness checked | 1.0 |
+| Project files on filesystem | 1.0 |
+| Reachable fallback Nuget feed count | 2.0 |
+| Resolved assembly conflicts | 7.0 |
+| Resource extraction enabled | 0.0 |
+| Restored .NET framework variants | 0.0 |
+| Solution files on filesystem | 1.0 |
+| Source files generated | 0.0 |
+| Source files on filesystem | 1.0 |
+| Successfully ran fallback nuget restore | 1.0 |
+| Unresolved references | 0.0 |
+| UseWPF set | 0.0 |
+| UseWindowsForms set | 0.0 |
+| WebView extraction enabled | 1.0 |

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/CompilationInfo.ql

@@ -0,0 +1,15 @@
+import csharp
+import semmle.code.csharp.commons.Diagnostics
+
+query predicate compilationInfo(string key, float value) {
+  key != "Resolved references" and
+  not key.matches("Compiler diagnostic count for%") and
+  exists(Compilation c, string infoKey, string infoValue | infoValue = c.getInfo(infoKey) |
+    key = infoKey and
+    value = infoValue.toFloat()
+    or
+    not exists(infoValue.toFloat()) and
+    key = infoKey + ": " + infoValue and
+    value = 1
+  )
+}

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/diagnostics.expected

@@ -0,0 +1,42 @@
+{
+  "markdownMessage": "C# analysis with build-mode 'none' completed.",
+  "severity": "unknown",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/buildless/complete",
+    "name": "C# analysis with build-mode 'none' completed"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": false,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "C# with build-mode set to 'none'. This means that all C# source in the working directory will be scanned, with build tools, such as Nuget and Dotnet CLIs, only contributing information about external dependencies.",
+  "severity": "note",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/buildless/mode-active",
+    "name": "C# with build-mode set to 'none'"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
+  "severity": "warning",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/buildless/unreachable-feed",
+    "name": "Found unreachable Nuget feed in C# analysis with build-mode 'none'"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/proj/Program.cs

@@ -0,0 +1,6 @@
+class Program
+{
+    static void Main(string[] args)
+    {
+    }
+}

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/proj/nuget.config

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="x" value="https://www.nuget.org/api/v2/" />
+  </packageSources>
+</configuration>

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/proj/proj.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFrameworks>net8.0</TargetFrameworks>
+  </PropertyGroup>
+
+  <Target Name="DeleteBinObjFolders" BeforeTargets="Clean">
+    <RemoveDir Directories=".\bin" />
+    <RemoveDir Directories=".\obj" />
+  </Target>
+
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+  </ItemGroup>
+</Project>

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/standalone.sln

@@ -0,0 +1,19 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.5.002.0
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "proj", "proj\proj.csproj", "{6ED00460-7666-4AE9-A405-4B6C8B02279A}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {4ED55A1C-066C-43DF-B32E-7EAA035985EE}
+	EndGlobalSection
+EndGlobal

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/test.py

@@ -0,0 +1,14 @@
+from create_database_utils import *
+from diagnostics_test_utils import *
+import os
+
+# os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK"] = "true"        # Nuget feed check is enabled by default
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT"] = "1"     # 1ms, the GET request should fail with such short timeout
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT"] = "1"       # Limit the count of checks to 1
+
+# Making sure the reachability test succeeds when doing a fallback restore:
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_TIMEOUT"] = "1000"
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_LIMIT"] = "5"
+
+run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])
+check_diagnostics()

csharp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-all
-version: 1.0.0-dev
+version: 0.10.2-dev
 groups: csharp
 dbscheme: semmlecode.csharp.dbscheme
 extractor: csharp

csharp/ql/src/experimental/Security Features/JsonWebTokenHandler/JsonWebTokenHandlerLib.qll

@@ -125,26 +125,17 @@ class TokenValidationParametersProperty extends Property {
 predicate callableHasAReturnStmtAndAlwaysReturnsTrue(Callable c) {
   c.getReturnType() instanceof BoolType and
   not callableMayThrowException(c) and
-  forall(ReturnStmt rs | rs.getEnclosingCallable() = c |
+  forex(ReturnStmt rs | rs.getEnclosingCallable() = c |
     rs.getNumberOfChildren() = 1 and
     isExpressionAlwaysTrue(rs.getChildExpr(0))
-  ) and
-  exists(ReturnStmt rs | rs.getEnclosingCallable() = c)
+  )
 }
 
 /**
  * Holds if the lambda expression `le` always returns true
  */
 predicate lambdaExprReturnsOnlyLiteralTrue(AnonymousFunctionExpr le) {
-  le.getExpressionBody().(BoolLiteral).getBoolValue() = true
-  or
-  // special scenarios where the expression is not a `BoolLiteral`, but it will evaluatue to `true`
-  exists(Expr e | le.getExpressionBody() = e |
-    not e instanceof Call and
-    not e instanceof Literal and
-    e.getType() instanceof BoolType and
-    e.getValue() = "true"
-  )
+  isExpressionAlwaysTrue(le.getExpressionBody())
 }
 
 class CallableAlwaysReturnsTrue extends Callable {
@@ -152,12 +143,6 @@ class CallableAlwaysReturnsTrue extends Callable {
     callableHasAReturnStmtAndAlwaysReturnsTrue(this)
     or
     lambdaExprReturnsOnlyLiteralTrue(this)
-    or
-    exists(AnonymousFunctionExpr le, Call call, Callable callable | this = le |
-      callable.getACall() = call and
-      call = le.getExpressionBody() and
-      callableHasAReturnStmtAndAlwaysReturnsTrue(callable)
-    )
   }
 }
 
@@ -171,32 +156,6 @@ predicate callableOnlyThrowsArgumentNullException(Callable c) {
   )
 }
 
-/**
- * A specialization of `CallableAlwaysReturnsTrue` that takes into consideration exceptions being thrown for higher precision.
- */
-class CallableAlwaysReturnsTrueHigherPrecision extends CallableAlwaysReturnsTrue {
-  CallableAlwaysReturnsTrueHigherPrecision() {
-    callableOnlyThrowsArgumentNullException(this) and
-    (
-      forall(Call call, Callable callable | call.getEnclosingCallable() = this |
-        callable.getACall() = call and
-        callable instanceof CallableAlwaysReturnsTrueHigherPrecision
-      )
-      or
-      exists(AnonymousFunctionExpr le, Call call, CallableAlwaysReturnsTrueHigherPrecision cat |
-        this = le
-      |
-        le.canReturn(call) and
-        cat.getACall() = call
-      )
-      or
-      exists(LambdaExpr le | le = this |
-        le.getBody() instanceof CallableAlwaysReturnsTrueHigherPrecision
-      )
-    )
-  }
-}
-
 /**
  * A callable that returns a `string` and has a `string` as 1st argument
  */

csharp/ql/src/experimental/Security Features/JsonWebTokenHandler/delegated-security-validations-always-return-true.ql

@@ -17,9 +17,7 @@ import DataFlow
 import JsonWebTokenHandlerLib
 import semmle.code.csharp.commons.QualifiedName
 
-from
-  TokenValidationParametersProperty p, CallableAlwaysReturnsTrueHigherPrecision e, string qualifier,
-  string name
+from TokenValidationParametersProperty p, CallableAlwaysReturnsTrue e, string qualifier, string name
 where e = p.getAnAssignedValue() and p.hasFullyQualifiedName(qualifier, name)
 select e,
   "JsonWebTokenHandler security-sensitive property $@ is being delegated to this callable that always returns \"true\".",

csharp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-queries
-version: 1.0.0-dev
+version: 0.8.17-dev
 groups:
   - csharp
   - queries

csharp/tools/tracing-config.lua

@@ -179,8 +179,6 @@ function RegisterExtractorPack(id)
     end
 
     local windowsMatchers = {
-        CreatePatternMatcher({ '^semmle%.extraction%.csharp%.standalone%.exe$' },
-            MatchCompilerName, nil, { trace = false }),
         DotnetMatcherBuild,
         MsBuildMatcher,
         CreatePatternMatcher({ '^csc.*%.exe$' }, MatchCompilerName, extractor, {
@@ -222,9 +220,6 @@ function RegisterExtractorPack(id)
         end
     }
     local posixMatchers = {
-        -- The compiler name is case sensitive on Linux and lower cased on MacOS
-        CreatePatternMatcher({ '^semmle%.extraction%.csharp%.standalone$', '^Semmle%.Extraction%.CSharp%.Standalone$' },
-            MatchCompilerName, nil, { trace = false }),
         DotnetMatcherBuild,
         CreatePatternMatcher({ '^mcs%.exe$', '^csc%.exe$' }, MatchCompilerName,
             extractor, {

docs/codeql/ql-language-reference/annotations.rst

@@ -446,7 +446,7 @@ The ``pragma[assume_small_delta]`` annotation has no effect and can be safely re
 Language pragmas
 ================
 
-**Available for**: |classes|, |characteristic predicates|, |member predicates|, |non-member predicates|
+**Available for**: |modules|, |classes|, |characteristic predicates|, |member predicates|, |non-member predicates|
 
 ``language[monotonicAggregates]``
 ---------------------------------

go/ql/consistency-queries/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql-go-consistency-queries
-version: 1.0.0-dev
+version: 0.0.16-dev
 groups:
   - go
   - queries

go/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/go-all
-version: 1.0.0-dev
+version: 0.8.2-dev
 groups: go
 dbscheme: go.dbscheme
 extractor: go

go/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/go-queries
-version: 1.0.0-dev
+version: 0.7.17-dev
 groups:
   - go
   - queries

java/documentation/library-coverage/coverage.csv

@@ -87,7 +87,7 @@ java.rmi,,,71,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,71,
 java.security,21,,543,,,11,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,539,4
 java.sql,15,1,303,,,,1,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,,9,,,,,,,,,1,,,,303,
 java.text,,,134,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,134,
-java.time,,,476,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,388,88
+java.time,,,123,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,35,88
 java.util,47,2,1218,,,,,,,,,1,,,,,,,,,,,34,,,,2,,,,5,2,,1,2,,,,,,,,,,,,,2,,,704,514
 javafx.scene.web,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,
 javax.accessibility,,,31,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,31,

java/documentation/library-coverage/coverage.rst

@@ -18,10 +18,10 @@ Java framework & library support
    `Google Guava <https://guava.dev/>`_,``com.google.common.*``,,730,43,9,,,,,
    JBoss Logging,``org.jboss.logging``,,,324,,,,,,
    `JSON-java <https://github.com/stleary/JSON-java>`_,``org.json``,,236,,,,,,,
-   Java Standard Library,``java.*``,10,4620,240,80,,9,,,26
+   Java Standard Library,``java.*``,10,4267,240,80,,9,,,26
    Java extensions,"``javax.*``, ``jakarta.*``",69,3257,85,5,4,2,1,1,4
    Kotlin Standard Library,``kotlin*``,,1849,16,14,,,,,2
    `Spring <https://spring.io/>`_,``org.springframework.*``,38,481,122,5,,28,14,,35
    Others,"``actions.osgi``, ``antlr``, ``ch.ethz.ssh2``, ``cn.hutool.core.codec``, ``com.alibaba.druid.sql``, ``com.alibaba.fastjson2``, ``com.amazonaws.auth``, ``com.auth0.jwt.algorithms``, ``com.azure.identity``, ``com.esotericsoftware.kryo.io``, ``com.esotericsoftware.kryo5.io``, ``com.fasterxml.jackson.core``, ``com.fasterxml.jackson.databind``, ``com.google.gson``, ``com.hubspot.jinjava``, ``com.jcraft.jsch``, ``com.microsoft.sqlserver.jdbc``, ``com.mitchellbosecke.pebble``, ``com.mongodb``, ``com.opensymphony.xwork2``, ``com.rabbitmq.client``, ``com.sshtools.j2ssh.authentication``, ``com.sun.crypto.provider``, ``com.sun.jndi.ldap``, ``com.sun.net.httpserver``, ``com.sun.net.ssl``, ``com.sun.rowset``, ``com.sun.security.auth.module``, ``com.sun.security.ntlm``, ``com.sun.security.sasl.digest``, ``com.thoughtworks.xstream``, ``com.trilead.ssh2``, ``com.unboundid.ldap.sdk``, ``com.zaxxer.hikari``, ``flexjson``, ``freemarker.cache``, ``freemarker.template``, ``groovy.lang``, ``groovy.text``, ``groovy.util``, ``hudson``, ``io.jsonwebtoken``, ``io.netty.bootstrap``, ``io.netty.buffer``, ``io.netty.channel``, ``io.netty.handler.codec``, ``io.netty.handler.ssl``, ``io.netty.handler.stream``, ``io.netty.resolver``, ``io.netty.util``, ``javafx.scene.web``, ``jenkins``, ``jodd.json``, ``liquibase.database.jvm``, ``liquibase.statement.core``, ``net.schmizz.sshj``, ``net.sf.json``, ``net.sf.saxon.s9api``, ``ognl``, ``okhttp3``, ``org.acegisecurity``, ``org.antlr.runtime``, ``org.apache.commons.codec``, ``org.apache.commons.compress.archivers.tar``, ``org.apache.commons.exec``, ``org.apache.commons.httpclient.util``, ``org.apache.commons.jelly``, ``org.apache.commons.jexl2``, ``org.apache.commons.jexl3``, ``org.apache.commons.lang``, ``org.apache.commons.logging``, ``org.apache.commons.net``, ``org.apache.commons.ognl``, ``org.apache.cxf.catalog``, ``org.apache.cxf.common.classloader``, ``org.apache.cxf.common.jaxb``, ``org.apache.cxf.common.logging``, ``org.apache.cxf.configuration.jsse``, ``org.apache.cxf.helpers``, ``org.apache.cxf.resource``, ``org.apache.cxf.staxutils``, ``org.apache.cxf.tools.corba.utils``, ``org.apache.cxf.tools.util``, ``org.apache.cxf.transform``, ``org.apache.directory.ldap.client.api``, ``org.apache.hadoop.fs``, ``org.apache.hadoop.hive.metastore``, ``org.apache.hadoop.hive.ql.exec``, ``org.apache.hadoop.hive.ql.metadata``, ``org.apache.hc.client5.http.async.methods``, ``org.apache.hc.client5.http.classic.methods``, ``org.apache.hc.client5.http.fluent``, ``org.apache.hive.hcatalog.templeton``, ``org.apache.ibatis.jdbc``, ``org.apache.ibatis.mapping``, ``org.apache.log4j``, ``org.apache.shiro.codec``, ``org.apache.shiro.jndi``, ``org.apache.shiro.mgt``, ``org.apache.sshd.client.session``, ``org.apache.struts.beanvalidation.validation.interceptor``, ``org.apache.struts2``, ``org.apache.tools.ant``, ``org.apache.tools.zip``, ``org.apache.velocity.app``, ``org.apache.velocity.runtime``, ``org.codehaus.cargo.container.installer``, ``org.codehaus.groovy.control``, ``org.dom4j``, ``org.eclipse.jetty.client``, ``org.fusesource.leveldbjni``, ``org.geogebra.web.full.main``, ``org.gradle.api.file``, ``org.hibernate``, ``org.influxdb``, ``org.jdbi.v3.core``, ``org.jenkins.ui.icon``, ``org.jenkins.ui.symbol``, ``org.jooq``, ``org.keycloak.models.map.storage``, ``org.kohsuke.stapler``, ``org.mvel2``, ``org.openjdk.jmh.runner.options``, ``org.owasp.esapi``, ``org.pac4j.jwt.config.encryption``, ``org.pac4j.jwt.config.signature``, ``org.scijava.log``, ``org.slf4j``, ``org.thymeleaf``, ``org.w3c.dom``, ``org.xml.sax``, ``org.xmlpull.v1``, ``org.yaml.snakeyaml``, ``play.libs.ws``, ``play.mvc``, ``ratpack.core.form``, ``ratpack.core.handling``, ``ratpack.core.http``, ``ratpack.exec``, ``ratpack.form``, ``ratpack.func``, ``ratpack.handling``, ``ratpack.http``, ``ratpack.util``, ``retrofit2``, ``sun.awt``, ``sun.jvmstat.perfdata.monitor.protocol.local``, ``sun.jvmstat.perfdata.monitor.protocol.rmi``, ``sun.management.spi``, ``sun.misc``, ``sun.net.ftp``, ``sun.net.www.protocol.http``, ``sun.nio.ch``, ``sun.security.acl``, ``sun.security.jgss.krb5``, ``sun.security.krb5``, ``sun.security.pkcs``, ``sun.security.pkcs11``, ``sun.security.provider``, ``sun.security.ssl``, ``sun.security.x509``, ``sun.tools.jconsole``, ``sun.util.logging.internal``",131,10596,893,125,6,22,18,,208
-   Totals,,310,25483,2569,338,16,128,33,1,409
+   Totals,,310,25130,2569,338,16,128,33,1,409
 

java/ql/automodel/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-automodel-queries
-version: 1.0.0-dev
+version: 0.0.24-dev
 groups:
     - java
     - automodel

java/ql/automodel/test/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-automodel-tests
-version: 1.0.0-dev
+version: 0.0.1-dev
 groups:
     - java
     - automodel

java/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-all
-version: 1.0.0-dev
+version: 0.11.1-dev
 groups: java
 dbscheme: config/semmlecode.dbscheme
 extractor: java

java/ql/lib/semmle/code/java/dataflow/ExternalFlow.qll

@@ -413,25 +413,28 @@ private string paramsStringQualified(Callable c) {
 }
 
 private Element interpretElement0(
-  string package, string type, boolean subtypes, string name, string signature
+  string package, string type, boolean subtypes, string name, string signature, boolean isExact
 ) {
   elementSpec(package, type, subtypes, name, signature, _) and
   (
     exists(Member m |
       (
-        result = m
+        result = m and isExact = true
         or
-        subtypes = true and result.(SrcMethod).overridesOrInstantiates+(m)
+        subtypes = true and result.(SrcMethod).overridesOrInstantiates+(m) and isExact = false
       ) and
       m.hasQualifiedName(package, type, name)
     |
-      signature = "" or
-      paramsStringQualified(m) = signature or
+      signature = ""
+      or
+      paramsStringQualified(m) = signature
+      or
       paramsString(m) = signature
     )
     or
     exists(RefType t |
       t.hasQualifiedName(package, type) and
+      isExact = false and
       (if subtypes = true then result.(SrcRefType).getASourceSupertype*() = t else result = t) and
       name = "" and
       signature = ""
@@ -442,13 +445,16 @@ private Element interpretElement0(
 /** Gets the source/sink/summary/neutral element corresponding to the supplied parameters. */
 cached
 Element interpretElement(
-  string package, string type, boolean subtypes, string name, string signature, string ext
+  string package, string type, boolean subtypes, string name, string signature, string ext,
+  boolean isExact
 ) {
   elementSpec(package, type, subtypes, name, signature, ext) and
-  exists(Element e | e = interpretElement0(package, type, subtypes, name, signature) |
-    ext = "" and result = e
+  exists(Element e, boolean isExact0 |
+    e = interpretElement0(package, type, subtypes, name, signature, isExact0)
+  |
+    ext = "" and result = e and isExact = isExact0
     or
-    ext = "Annotated" and result.(Annotatable).getAnAnnotation().getType() = e
+    ext = "Annotated" and result.(Annotatable).getAnAnnotation().getType() = e and isExact = false
   )
 }
 
@@ -538,13 +544,13 @@ predicate sinkNode(Node node, string kind) { sinkNode(node, kind, _) }
 
 // adapter class for converting Mad summaries to `SummarizedCallable`s
 private class SummarizedCallableAdapter extends SummarizedCallable {
-  SummarizedCallableAdapter() { summaryElement(this, _, _, _, _, _) }
+  SummarizedCallableAdapter() { summaryElement(this, _, _, _, _, _, _) }
 
   private predicate relevantSummaryElementManual(
     string input, string output, string kind, string model
   ) {
     exists(Provenance provenance |
-      summaryElement(this, input, output, kind, provenance, model) and
+      summaryElement(this, input, output, kind, provenance, model, _) and
       provenance.isManual()
     )
   }
@@ -553,11 +559,11 @@ private class SummarizedCallableAdapter extends SummarizedCallable {
     string input, string output, string kind, string model
   ) {
     exists(Provenance provenance |
-      summaryElement(this, input, output, kind, provenance, model) and
+      summaryElement(this, input, output, kind, provenance, model, _) and
       provenance.isGenerated()
     ) and
     not exists(Provenance provenance |
-      neutralElement(this, "summary", provenance) and
+      neutralElement(this, "summary", provenance, _) and
       provenance.isManual()
     )
   }
@@ -576,18 +582,23 @@ private class SummarizedCallableAdapter extends SummarizedCallable {
   }
 
   override predicate hasProvenance(Provenance provenance) {
-    summaryElement(this, _, _, _, provenance, _)
+    summaryElement(this, _, _, _, provenance, _, _)
   }
+
+  override predicate hasExactModel() { summaryElement(this, _, _, _, _, _, true) }
 }
 
 // adapter class for converting Mad neutrals to `NeutralCallable`s
 private class NeutralCallableAdapter extends NeutralCallable {
   string kind;
   string provenance_;
+  boolean exact;
 
-  NeutralCallableAdapter() { neutralElement(this, kind, provenance_) }
+  NeutralCallableAdapter() { neutralElement(this, kind, provenance_, exact) }
 
   override string getKind() { result = kind }
 
   override predicate hasProvenance(Provenance provenance) { provenance = provenance_ }
+
+  override predicate hasExactModel() { exact = true }
 }

java/ql/lib/semmle/code/java/dataflow/FlowSummary.qll

@@ -135,6 +135,8 @@ private class SummarizedSyntheticCallableAdapter extends SummarizedCallable, TSy
       model = sc
     )
   }
+
+  override predicate hasExactModel() { any() }
 }
 
 deprecated class RequiredSummaryComponentStack = Impl::Private::RequiredSummaryComponentStack;

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowDispatch.qll

@@ -19,7 +19,21 @@ private module DispatchImpl {
     )
   }
 
+  private predicate hasExactManualModel(Call c, Callable tgt) {
+    tgt = c.getCallee().getSourceDeclaration() and
+    (
+      exists(Impl::Public::SummarizedCallable sc |
+        sc.getACall() = c and sc.hasExactModel() and sc.hasManualModel()
+      )
+      or
+      exists(Impl::Public::NeutralSummaryCallable nc |
+        nc.getACall() = c and nc.hasExactModel() and nc.hasManualModel()
+      )
+    )
+  }
+
   private Callable sourceDispatch(Call c) {
+    not hasExactManualModel(c, result) and
     result = VirtualDispatch::viableCallable(c) and
     if VirtualDispatch::lowConfidenceDispatchTarget(c, result)
     then not hasHighConfidenceTarget(c)
@@ -122,12 +136,18 @@ private module DispatchImpl {
     mayBenefitFromCallContext(call.asCall(), _, _)
   }
 
+  bindingset[call, tgt]
+  pragma[inline_late]
+  private predicate viableCallableFilter(DataFlowCall call, DataFlowCallable tgt) {
+    tgt = viableCallable(call)
+  }
+
   /**
    * Gets a viable dispatch target of `call` in the context `ctx`. This is
    * restricted to those `call`s for which a context might make a difference.
    */
   DataFlowCallable viableImplInCallContext(DataFlowCall call, DataFlowCall ctx) {
-    result = viableCallable(call) and
+    viableCallableFilter(call, result) and
     exists(int i, Callable c, Method def, RefType t, boolean exact, MethodCall ma |
       ma = call.asCall() and
       mayBenefitFromCallContext(ma, c, i) and

java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll

@@ -131,7 +131,7 @@ private predicate relatedArgSpec(Callable c, string spec) {
     sourceModel(namespace, type, subtypes, name, signature, ext, spec, _, _, _) or
     sinkModel(namespace, type, subtypes, name, signature, ext, spec, _, _, _)
   |
-    c = interpretElement(namespace, type, subtypes, name, signature, ext)
+    c = interpretElement(namespace, type, subtypes, name, signature, ext, _)
   )
 }
 
@@ -202,7 +202,7 @@ module SourceSinkInterpretationInput implements
       sourceModel(namespace, type, subtypes, name, signature, ext, originalOutput, kind, provenance,
         madId) and
       model = "MaD:" + madId.toString() and
-      baseSource = interpretElement(namespace, type, subtypes, name, signature, ext) and
+      baseSource = interpretElement(namespace, type, subtypes, name, signature, ext, _) and
       (
         e = baseSource and output = originalOutput
         or
@@ -221,7 +221,7 @@ module SourceSinkInterpretationInput implements
       sinkModel(namespace, type, subtypes, name, signature, ext, originalInput, kind, provenance,
         madId) and
       model = "MaD:" + madId.toString() and
-      baseSink = interpretElement(namespace, type, subtypes, name, signature, ext) and
+      baseSink = interpretElement(namespace, type, subtypes, name, signature, ext, _) and
       (
         e = baseSink and originalInput = input
         or
@@ -310,7 +310,7 @@ module Private {
      */
     predicate summaryElement(
       Input::SummarizedCallableBase c, string input, string output, string kind, string provenance,
-      string model
+      string model, boolean isExact
     ) {
       exists(
         string namespace, string type, boolean subtypes, string name, string signature, string ext,
@@ -320,7 +320,7 @@ module Private {
         summaryModel(namespace, type, subtypes, name, signature, ext, originalInput, originalOutput,
           kind, provenance, madId) and
         model = "MaD:" + madId.toString() and
-        baseCallable = interpretElement(namespace, type, subtypes, name, signature, ext) and
+        baseCallable = interpretElement(namespace, type, subtypes, name, signature, ext, isExact) and
         (
           c.asCallable() = baseCallable and input = originalInput and output = originalOutput
           or
@@ -336,10 +336,12 @@ module Private {
      * Holds if a neutral model exists for `c` of kind `kind`
      * and with provenance `provenance`.
      */
-    predicate neutralElement(Input::SummarizedCallableBase c, string kind, string provenance) {
+    predicate neutralElement(
+      Input::SummarizedCallableBase c, string kind, string provenance, boolean isExact
+    ) {
       exists(string namespace, string type, string name, string signature |
         neutralModel(namespace, type, name, signature, kind, provenance) and
-        c.asCallable() = interpretElement(namespace, type, false, name, signature, "")
+        c.asCallable() = interpretElement(namespace, type, false, name, signature, "", isExact)
       )
     }
   }

java/ql/src/Security/CWE/CWE-501/TrustBoundaryViolation.ql

@@ -16,5 +16,5 @@ import TrustBoundaryFlow::PathGraph
 
 from TrustBoundaryFlow::PathNode source, TrustBoundaryFlow::PathNode sink
 where TrustBoundaryFlow::flowPath(source, sink)
-select sink.getNode(), sink, source,
-  "This servlet reads data from a remote source and writes it to a session variable."
+select sink.getNode(), source, sink,
+  "This servlet reads data from a $@ and writes it to a session variable.", source, "remote source"

java/ql/src/Security/CWE/CWE-611/XXE.qhelp

@@ -28,10 +28,9 @@ This improves security but the code will still be at risk of denial of service a
 Protection against denial of service attacks may also be implemented by setting entity expansion limits, which is done
 by default in recent JDK and JRE implementations.
 
-Because there are many different ways to disable external entity retrieval with varying support between different providers,
-in this query we choose to specifically check for the <a href="https://cheatsheetseries.owasp.org/cheatsheets/XML_External_Entity_Prevention_Cheat_Sheet.html#java">OWASP recommended way</a>
-to disable external entity retrieval for a particular parser. There may be other ways of making a particular parser safe
-which deviate from these guidelines, in which case this query will continue to flag the parser as potentially dangerous.
+We recommend visiting OWASP's <a href="https://cheatsheetseries.owasp.org/cheatsheets/XML_External_Entity_Prevention_Cheat_Sheet.html#java">XML Entity Prevention Cheat Sheet</a>,
+finding the specific XML parser, and applying the mitigation listed there. Other mitigations might be sufficient in some cases, but manual verification will be needed,
+as the query will continue to flag the parser as potentially dangerous.
 </p>
 </recommendation>
 

java/ql/src/change-notes/2024-05-23-trusted-boundary-violation.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The alert message for the query "Trust boundary violation" (`java/trust-boundary-violation`) has been updated to include a link to the remote source.

java/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-queries
-version: 1.0.0-dev
+version: 0.8.17-dev
 groups:
   - java
   - queries

java/ql/src/utils/modeleditor/ModelEditor.qll

@@ -77,7 +77,7 @@ class Endpoint extends Callable {
   predicate isNeutral() {
     exists(string namespace, string type, string name, string signature |
       neutralModel(namespace, type, name, signature, _, _) and
-      this = interpretElement(namespace, type, false, name, signature, "")
+      this = interpretElement(namespace, type, false, name, signature, "", _)
     )
   }
 

javascript/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/javascript-all
-version: 1.0.0-dev
+version: 0.9.2-dev
 groups: javascript
 dbscheme: semmlecode.javascript.dbscheme
 extractor: javascript

javascript/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/javascript-queries
-version: 1.0.0-dev
+version: 0.8.17-dev
 groups:
   - javascript
   - queries

misc/suite-helpers/qlpack.yml

@@ -1,4 +1,4 @@
 name: codeql/suite-helpers
-version: 1.0.0-dev
+version: 0.7.17-dev
 groups: shared
 warnOnImplicitThis: true

python/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/python-all
-version: 1.0.0-dev
+version: 0.12.2-dev
 groups: python
 dbscheme: semmlecode.python.dbscheme
 extractor: python

python/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/python-queries
-version: 1.0.0-dev
+version: 0.9.17-dev
 groups:
   - python
   - queries

ql/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/ql
-version: 1.0.0-dev
+version: 0.1.0-dev
 groups:
   - ql
   - queries

ruby/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/ruby-all
-version: 1.0.0-dev
+version: 0.9.2-dev
 groups: ruby
 extractor: ruby
 dbscheme: ruby.dbscheme

ruby/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/ruby-queries
-version: 1.0.0-dev
+version: 0.8.17-dev
 groups:
   - ruby
   - queries

shared/controlflow/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/controlflow
-version: 1.0.0-dev
+version: 0.1.17-dev
 groups: shared
 library: true
 dependencies:

shared/dataflow/codeql/dataflow/internal/DataFlowImplCommon.qll

@@ -659,6 +659,7 @@ module MakeImplCommon<LocationSig Location, InputSig<Location> Lang> {
     predicate expectsContentCached(Node n, ContentSet c) { expectsContent(n, c) }
 
     cached
+    pragma[no_dynamic_join_order]
     predicate isUnreachableInCallCached(Node n, DataFlowCall call) { isUnreachableInCall(n, call) }
 
     cached

shared/dataflow/codeql/dataflow/internal/FlowSummaryImpl.qll

@@ -253,6 +253,13 @@ module Make<
        * that has provenance `provenance`.
        */
       predicate hasProvenance(Provenance provenance) { provenance = "manual" }
+
+      /**
+       * Holds if there exists a model for which this callable is an exact
+       * match, that is, no overriding was used to identify this callable from
+       * the model.
+       */
+      predicate hasExactModel() { none() }
     }
 
     final private class NeutralCallableFinal = NeutralCallable;
@@ -292,6 +299,13 @@ module Make<
        * Gets the kind of the neutral.
        */
       abstract string getKind();
+
+      /**
+       * Holds if there exists a model for which this callable is an exact
+       * match, that is, no overriding was used to identify this callable from
+       * the model.
+       */
+      predicate hasExactModel() { none() }
     }
   }
 

shared/dataflow/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/dataflow
-version: 1.0.0-dev
+version: 0.2.8-dev
 groups: shared
 library: true
 dependencies:

shared/mad/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/mad
-version: 1.0.0-dev
+version: 0.2.17-dev
 groups: shared
 library: true
 dependencies:

shared/rangeanalysis/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/rangeanalysis
-version: 1.0.0-dev
+version: 0.0.16-dev
 groups: shared
 library: true
 dependencies:

shared/regex/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/regex
-version: 1.0.0-dev
+version: 0.2.17-dev
 groups: shared
 library: true
 dependencies:

shared/ssa/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/ssa
-version: 1.0.0-dev
+version: 0.2.17-dev
 groups: shared
 library: true
 dependencies:

shared/threat-models/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/threat-models
-version: 1.0.0-dev
+version: 0.0.16-dev
 library: true
 groups: shared
 dataExtensions:

shared/tutorial/qlpack.yml

@@ -1,7 +1,7 @@
 name: codeql/tutorial
 description: Library for the CodeQL detective tutorials, helping new users learn to
   write CodeQL queries.
-version: 1.0.0-dev
+version: 0.2.17-dev
 groups: shared
 library: true
 warnOnImplicitThis: true

shared/typeflow/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/typeflow
-version: 1.0.0-dev
+version: 0.0.4-dev
 groups: shared
 library: true
 dependencies:

shared/typetracking/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/typetracking
-version: 1.0.0-dev
+version: 0.2.17-dev
 groups: shared
 library: true
 dependencies:

shared/typos/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/typos
-version: 1.0.0-dev
+version: 0.2.17-dev
 groups: shared
 library: true
 warnOnImplicitThis: true

shared/util/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/util
-version: 1.0.0-dev
+version: 0.2.17-dev
 groups: shared
 library: true
 dependencies: null

shared/xml/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/xml
-version: 1.0.0-dev
+version: 0.0.4-dev
 groups: shared
 library: true
 dependencies:

shared/yaml/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/yaml
-version: 1.0.0-dev
+version: 0.2.17-dev
 groups: shared
 library: true
 warnOnImplicitThis: true

swift/README.md

@@ -16,7 +16,14 @@ brew install bazelisk
 then from the `ql` directory run
 
 ```bash
-bazel run //swift:create-extractor-pack    # --cpu=darwin_x86_64 # Uncomment on Arm-based Macs
+bazel run //swift:create-extractor-pack
+```
+
+If you are running on macOS and you encounter errors mentioning `XXX is unavailable: introduced in macOS YY.ZZ`,
+you will need to run this from the root of your `codeql` checkout:
+
+```bash
+echo common --macos_sdk_version=$(sw_vers --productVersion) >> local.bazelrc
 ```
 
 which will install `swift/extractor-pack`.

swift/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/swift-all
-version: 1.0.0-dev
+version: 0.3.17-dev
 groups: swift
 extractor: swift
 dbscheme: swift.dbscheme

swift/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/swift-queries
-version: 1.0.0-dev
+version: 0.3.17-dev
 groups:
   - swift
   - queries