Merge branch 'main' into criemen/bazel-csharp

.bazelrc

@@ -11,7 +11,8 @@ common --override_module=semmle_code=%workspace%/misc/bazel/semmle_code_stub
 build --repo_env=CC=clang --repo_env=CXX=clang++
 
 build:linux --cxxopt=-std=c++20
-build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64
+# we currently cannot built the swift extractor for ARM
+build:macos --cxxopt=-std=c++20 --copt=-arch --copt=x86_64 --linkopt=-arch --linkopt=x86_64
 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor
 
 # this requires developer mode, but is required to have pack installer functioning

cpp/ql/lib/change-notes/2024-05-23-Version1.md

@@ -0,0 +1,4 @@
+---
+category: breaking
+---
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 0.13.2-dev
+version: 1.0.0-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/src/change-notes/2024-05-23-Version1.md

@@ -0,0 +1,4 @@
+---
+category: breaking
+---
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 0.9.13-dev
+version: 1.0.0-dev
 groups:
   - cpp
   - queries

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

@@ -60,6 +60,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         /// </summary>
         public const string FallbackNugetFeeds = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK";
 
+        /// <summary>
+        /// Controls whether to include NuGet feeds from nuget.config files in the fallback restore logic.
+        /// </summary>
+        public const string AddNugetConfigFeedsToFallback = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK_INCLUDE_NUGET_CONFIG_FEEDS";
+
         /// <summary>
         /// Specifies the path to the nuget executable to be used for package restoration.
         /// </summary>

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -98,12 +98,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             logger.LogInfo($"Checking NuGet feed responsiveness: {checkNugetFeedResponsiveness}");
             compilationInfoContainer.CompilationInfos.Add(("NuGet feed responsiveness checked", checkNugetFeedResponsiveness ? "1" : "0"));
 
+            HashSet<string>? explicitFeeds = null;
+
             try
             {
-                if (checkNugetFeedResponsiveness && !CheckFeeds())
+                if (checkNugetFeedResponsiveness && !CheckFeeds(out explicitFeeds))
                 {
                     // todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too.
-                    var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds();
+                    var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds(explicitFeeds);
                     return unresponsiveMissingPackageLocation is null
                         ? []
                         : [unresponsiveMissingPackageLocation];
@@ -163,7 +165,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             LogAllUnusedPackages(dependencies);
 
             var missingPackageLocation = checkNugetFeedResponsiveness
-                ? DownloadMissingPackagesFromSpecificFeeds()
+                ? DownloadMissingPackagesFromSpecificFeeds(explicitFeeds)
                 : DownloadMissingPackages();
 
             if (missingPackageLocation is not null)
@@ -173,13 +175,24 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return assemblyLookupLocations;
         }
 
-        private List<string> GetReachableFallbackNugetFeeds()
+        private List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNugetConfigs)
         {
             var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet();
             if (fallbackFeeds.Count == 0)
             {
                 fallbackFeeds.Add(PublicNugetOrgFeed);
-                logger.LogInfo($"No fallback Nuget feeds specified. Using default feed: {PublicNugetOrgFeed}");
+                logger.LogInfo($"No fallback Nuget feeds specified. Adding default feed: {PublicNugetOrgFeed}");
+
+                var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback);
+                logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}");
+
+                if (shouldAddNugetConfigFeeds && feedsFromNugetConfigs?.Count > 0)
+                {
+                    // There are some feeds in `feedsFromNugetConfigs` that have already been checked for reachability, we could skip those.
+                    // But we might use different responsiveness testing settings when we try them in the fallback logic, so checking them again is safer.
+                    fallbackFeeds.UnionWith(feedsFromNugetConfigs);
+                    logger.LogInfo($"Using Nuget feeds from nuget.config files as fallback feeds: {string.Join(", ", feedsFromNugetConfigs.OrderBy(f => f))}");
+                }
             }
 
             logger.LogInfo($"Checking fallback Nuget feed reachability on feeds: {string.Join(", ", fallbackFeeds.OrderBy(f => f))}");
@@ -194,6 +207,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 logger.LogInfo($"Reachable fallback Nuget feeds: {string.Join(", ", reachableFallbackFeeds.OrderBy(f => f))}");
             }
 
+            compilationInfoContainer.CompilationInfos.Add(("Reachable fallback Nuget feed count", reachableFallbackFeeds.Count.ToString()));
+
             return reachableFallbackFeeds;
         }
 
@@ -272,9 +287,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             compilationInfoContainer.CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
         }
 
-        private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds()
+        private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds(HashSet<string>? feedsFromNugetConfigs)
         {
-            var reachableFallbackFeeds = GetReachableFallbackNugetFeeds();
+            var reachableFallbackFeeds = GetReachableFallbackNugetFeeds(feedsFromNugetConfigs);
             if (reachableFallbackFeeds.Count > 0)
             {
                 return DownloadMissingPackages(fallbackNugetFeeds: reachableFallbackFeeds);
@@ -623,10 +638,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return (timeoutMilliSeconds, tryCount);
         }
 
-        private bool CheckFeeds()
+        private bool CheckFeeds(out HashSet<string> explicitFeeds)
         {
             logger.LogInfo("Checking Nuget feeds...");
-            var (explicitFeeds, allFeeds) = GetAllFeeds();
+            (explicitFeeds, var allFeeds) = GetAllFeeds();
 
             var excludedFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck)
                 .ToHashSet() ?? [];

csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs

@@ -11,26 +11,20 @@ namespace Semmle.Extraction.CSharp.Entities
     {
         internal readonly ConcurrentDictionary<string, int> messageCounts = new();
 
-        private static (string Cwd, string[] Args) settings;
+        private readonly string cwd;
-        private static int hashCode;
+        private readonly string[] args;
-
+        private readonly int hashCode;
-        public static (string Cwd, string[] Args) Settings
-        {
-            get { return settings; }
-            set
-            {
-                settings = value;
-                hashCode = settings.Cwd.GetHashCode();
-                for (var i = 0; i < settings.Args.Length; i++)
-                {
-                    hashCode = HashCode.Combine(hashCode, settings.Args[i].GetHashCode());
-                }
-            }
-        }
 
 #nullable disable warnings
         private Compilation(Context cx) : base(cx, null)
         {
+            cwd = cx.Extractor.Cwd;
+            args = cx.Extractor.Args;
+            hashCode = cwd.GetHashCode();
+            for (var i = 0; i < args.Length; i++)
+            {
+                hashCode = HashCode.Combine(hashCode, args[i].GetHashCode());
+            }
         }
 #nullable restore warnings
 
@@ -38,14 +32,14 @@ namespace Semmle.Extraction.CSharp.Entities
         {
             var assembly = Assembly.CreateOutputAssembly(Context);
 
-            trapFile.compilations(this, FileUtils.ConvertToUnix(Compilation.Settings.Cwd));
+            trapFile.compilations(this, FileUtils.ConvertToUnix(cwd));
             trapFile.compilation_assembly(this, assembly);
 
             // Arguments
             var expandedIndex = 0;
-            for (var i = 0; i < Compilation.Settings.Args.Length; i++)
+            for (var i = 0; i < args.Length; i++)
             {
-                var arg = Compilation.Settings.Args[i];
+                var arg = args[i];
                 trapFile.compilation_args(this, i, arg);
 
                 if (CommandLineExtensions.IsFileArgument(arg))

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs

@@ -97,7 +97,8 @@ namespace Semmle.Extraction.CSharp
             stopwatch.Start();
 
             var options = Options.CreateWithEnvironment(args);
-            Entities.Compilation.Settings = (Directory.GetCurrentDirectory(), options.CompilerArguments.ToArray());
+            var workingDirectory = Directory.GetCurrentDirectory();
+            var compilerArgs = options.CompilerArguments.ToArray();
 
             using var logger = MakeLogger(options.Verbosity, options.Console);
 
@@ -123,7 +124,7 @@ namespace Semmle.Extraction.CSharp
 
                 var compilerArguments = CSharpCommandLineParser.Default.Parse(
                     compilerVersion.ArgsWithResponse,
-                    Entities.Compilation.Settings.Cwd,
+                    workingDirectory,
                     compilerVersion.FrameworkPath,
                     compilerVersion.AdditionalReferenceDirectories
                     );
@@ -131,7 +132,7 @@ namespace Semmle.Extraction.CSharp
                 if (compilerArguments is null)
                 {
                     var sb = new StringBuilder();
-                    sb.Append("  Failed to parse command line: ").AppendList(" ", Entities.Compilation.Settings.Args);
+                    sb.Append("  Failed to parse command line: ").AppendList(" ", compilerArgs);
                     logger.Log(Severity.Error, sb.ToString());
                     ++analyser.CompilationErrors;
                     return ExitCode.Failed;
@@ -143,7 +144,7 @@ namespace Semmle.Extraction.CSharp
                     return ExitCode.Ok;
                 }
 
-                return AnalyseTracing(analyser, compilerArguments, options, canonicalPathCache, stopwatch);
+                return AnalyseTracing(workingDirectory, compilerArgs, analyser, compilerArguments, options, canonicalPathCache, stopwatch);
             }
             catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
             {
@@ -376,6 +377,8 @@ namespace Semmle.Extraction.CSharp
         }
 
         private static ExitCode AnalyseTracing(
+            string cwd,
+            string[] args,
             TracingAnalyser analyser,
             CSharpCommandLineArguments compilerArguments,
             Options options,
@@ -420,7 +423,7 @@ namespace Semmle.Extraction.CSharp
                             .WithMetadataImportOptions(MetadataImportOptions.All)
                         );
                 },
-                (compilation, options) => analyser.EndInitialize(compilerArguments, options, compilation),
+                (compilation, options) => analyser.EndInitialize(compilerArguments, options, compilation, cwd, args),
                 () => { });
         }
 

csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs

@@ -16,12 +16,10 @@ namespace Semmle.Extraction.CSharp
         public void Initialize(string outputPath, IEnumerable<(string, string)> compilationInfos, CSharpCompilation compilationIn, CommonOptions options)
         {
             compilation = compilationIn;
-            extractor = new StandaloneExtractor(outputPath, compilationInfos, Logger, PathTransformer, options);
+            extractor = new StandaloneExtractor(Directory.GetCurrentDirectory(), outputPath, compilationInfos, Logger, PathTransformer, options);
             this.options = options;
             LogExtractorInfo(Extraction.Extractor.Version);
             SetReferencePaths();
-
-            Entities.Compilation.Settings = (Directory.GetCurrentDirectory(), Array.Empty<string>());
         }
 
 #nullable disable warnings

csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs

@@ -38,13 +38,15 @@ namespace Semmle.Extraction.CSharp
         public void EndInitialize(
            CSharpCommandLineArguments commandLineArguments,
            CommonOptions options,
-           CSharpCompilation compilation)
+           CSharpCompilation compilation,
+           string cwd,
+           string[] args)
         {
             if (!init)
                 throw new InternalError("EndInitialize called without BeginInitialize returning true");
             this.options = options;
             this.compilation = compilation;
-            this.extractor = new TracingExtractor(GetOutputName(compilation, commandLineArguments), Logger, PathTransformer, options);
+            this.extractor = new TracingExtractor(cwd, args, GetOutputName(compilation, commandLineArguments), Logger, PathTransformer, options);
             LogDiagnostics();
 
             SetReferencePaths();

csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs

@@ -11,6 +11,8 @@ namespace Semmle.Extraction
     /// </summary>
     public abstract class Extractor
     {
+        public string Cwd { get; init; }
+        public string[] Args { get; init; }
         public abstract ExtractorMode Mode { get; }
         public string OutputPath { get; }
         public IEnumerable<CompilationInfo> CompilationInfos { get; }
@@ -20,12 +22,14 @@ namespace Semmle.Extraction
         /// </summary>
         /// <param name="logger">The object used for logging.</param>
         /// <param name="pathTransformer">The object used for path transformations.</param>
-        protected Extractor(string outputPath, IEnumerable<CompilationInfo> compilationInfos, ILogger logger, PathTransformer pathTransformer)
+        protected Extractor(string cwd, string[] args, string outputPath, IEnumerable<CompilationInfo> compilationInfos, ILogger logger, PathTransformer pathTransformer)
         {
             OutputPath = outputPath;
             Logger = logger;
             PathTransformer = pathTransformer;
             CompilationInfos = compilationInfos;
+            Cwd = cwd;
+            Args = args;
         }
 
         // Limit the number of error messages in the log file

csharp/extractor/Semmle.Extraction/Extractor/StandaloneExtractor.cs

@@ -12,7 +12,8 @@ namespace Semmle.Extraction
         /// </summary>
         /// <param name="logger">The object used for logging.</param>
         /// <param name="pathTransformer">The object used for path transformations.</param>
-        public StandaloneExtractor(string outputPath, IEnumerable<(string, string)> compilationInfos, ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(outputPath, compilationInfos, logger, pathTransformer)
+        public StandaloneExtractor(string cwd, string outputPath, IEnumerable<(string, string)> compilationInfos, ILogger logger, PathTransformer pathTransformer, CommonOptions options)
+            : base(cwd, [], outputPath, compilationInfos, logger, pathTransformer)
         {
             Mode = ExtractorMode.Standalone;
             if (options.QlTest)

csharp/extractor/Semmle.Extraction/Extractor/TracingExtractor.cs

@@ -1,4 +1,3 @@
-using System.Linq;
 using Semmle.Util.Logging;
 
 namespace Semmle.Extraction
@@ -13,7 +12,8 @@ namespace Semmle.Extraction
         /// <param name="outputPath">The name of the output DLL/EXE, or null if not specified (standalone extraction).</param>
         /// <param name="logger">The object used for logging.</param>
         /// <param name="pathTransformer">The object used for path transformations.</param>
-        public TracingExtractor(string outputPath, ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(outputPath, Enumerable.Empty<(string, string)>(), logger, pathTransformer)
+        public TracingExtractor(string cwd, string[] args, string outputPath, ILogger logger, PathTransformer pathTransformer, CommonOptions options)
+            : base(cwd, args, outputPath, [], logger, pathTransformer)
         {
             Mode = ExtractorMode.None;
             if (options.QlTest)

csharp/ql/integration-tests/all-platforms/standalone_resx/CompilationInfo.expected

@@ -3,6 +3,7 @@
 | Failed solution restore with package source error | 0.0 |
 | NuGet feed responsiveness checked | 1.0 |
 | Project files on filesystem | 1.0 |
+| Reachable fallback Nuget feed count | 1.0 |
 | Resource extraction enabled | 1.0 |
 | Restored .NET framework variants | 1.0 |
 | Restored projects through solution files | 0.0 |

csharp/ql/integration-tests/linux-only/compiler_args/CompilerArgs.expected

@@ -1,9 +1,14 @@
 | 0 | /noconfig |
 | 1 | /unsafe- |
 | 2 | /checked- |
+| 3 | /nowarn:1701,1702,1701,1702 |
 | 4 | /fullpaths |
 | 5 | /nostdlib+ |
+| 6 | /errorreport:prompt |
+| 7 | /warn:8 |
+| 8 | /define:TRACE;DEBUG;NET;NET8_0;NETCOREAPP;NET5_0_OR_GREATER;NET6_0_OR_GREATER;NET7_0_OR_GREATER;NET8_0_OR_GREATER;NETCOREAPP1_0_OR_GREATER;NETCOREAPP1_1_OR_GREATER;NETCOREAPP2_0_OR_GREATER;NETCOREAPP2_1_OR_GREATER;NETCOREAPP2_2_OR_GREATER;NETCOREAPP3_0_OR_GREATER;NETCOREAPP3_1_OR_GREATER |
 | 9 | /highentropyva+ |
+| 10 | /nullable:enable |
 | 11 | /reference:[...]/8.0.1/ref/net8.0/Microsoft.CSharp.dll |
 | 12 | /reference:[...]/8.0.1/ref/net8.0/Microsoft.VisualBasic.Core.dll |
 | 13 | /reference:[...]/8.0.1/ref/net8.0/Microsoft.VisualBasic.dll |
@@ -168,10 +173,24 @@
 | 172 | /reference:[...]/8.0.1/ref/net8.0/System.Xml.XPath.XDocument.dll |
 | 173 | /reference:[...]/8.0.1/ref/net8.0/WindowsBase.dll |
 | 174 | /debug+ |
+| 175 | /debug:portable |
+| 176 | /filealign:512 |
+| 177 | /generatedfilesout:obj/Debug/net8.0//generated |
 | 178 | /optimize- |
+| 179 | /out:obj/Debug/net8.0/test.dll |
+| 180 | /refout:obj/Debug/net8.0/refint/test.dll |
+| 181 | /target:exe |
 | 182 | /warnaserror- |
 | 183 | /utf8output |
 | 184 | /deterministic+ |
+| 185 | /sourcelink:obj/Debug/net8.0/test.sourcelink.json |
+| 186 | /langversion:12.0 |
+| 187 | /embed:Program.cs |
+| 188 | /embed:obj/Debug/net8.0/test.GlobalUsings.g.cs |
+| 189 | /embed:"obj/Debug/net8.0/.NETCoreApp,Version=v8.0.AssemblyAttributes.cs" |
+| 190 | /embed:obj/Debug/net8.0/test.AssemblyInfo.cs |
+| 191 | /analyzerconfig:/home/runner/work/semmle-code/semmle-code/.editorconfig |
+| 192 | /analyzerconfig:obj/Debug/net8.0/test.GeneratedMSBuildEditorConfig.editorconfig |
 | 193 | /analyzerconfig:[...]/8.0.101/Sdks/Microsoft.NET.Sdk/analyzers/build/config/analysislevel_8_default.globalconfig |
 | 194 | /analyzer:[...]/8.0.101/Sdks/Microsoft.NET.Sdk/targets/../analyzers/Microsoft.CodeAnalysis.CSharp.NetAnalyzers.dll |
 | 195 | /analyzer:[...]/8.0.101/Sdks/Microsoft.NET.Sdk/targets/../analyzers/Microsoft.CodeAnalysis.NetAnalyzers.dll |
@@ -185,3 +204,4 @@
 | 203 | obj/Debug/net8.0/test.GlobalUsings.g.cs |
 | 204 | obj/Debug/net8.0/.NETCoreApp,Version=v8.0.AssemblyAttributes.cs |
 | 205 | obj/Debug/net8.0/test.AssemblyInfo.cs |
+| 206 | /warnaserror+:NU1605,SYSLIB0011 |

csharp/ql/integration-tests/linux-only/compiler_args/CompilerArgs.ql

@@ -3,7 +3,8 @@ import semmle.code.csharp.commons.Compilation
 
 bindingset[arg]
 private string normalize(string arg) {
-  not exists(arg.indexOf(":")) and result = arg
+  (not exists(arg.indexOf(":")) or not exists(arg.indexOf("/8.0"))) and
+  result = arg
   or
   exists(int i, int j |
     i = arg.indexOf(":") and

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/CompilationInfo.expected

@@ -4,6 +4,7 @@
 | Fallback nuget restore | 1.0 |
 | NuGet feed responsiveness checked | 1.0 |
 | Project files on filesystem | 1.0 |
+| Reachable fallback Nuget feed count | 1.0 |
 | Resolved assembly conflicts | 7.0 |
 | Resource extraction enabled | 0.0 |
 | Restored .NET framework variants | 0.0 |

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.expected

@@ -3,6 +3,7 @@
 | Inherited Nuget feed count | 1.0 |
 | NuGet feed responsiveness checked | 1.0 |
 | Project files on filesystem | 1.0 |
+| Reachable fallback Nuget feed count | 1.0 |
 | Resolved assembly conflicts | 7.0 |
 | Resource extraction enabled | 0.0 |
 | Restored .NET framework variants | 0.0 |

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/Assemblies.expected

@@ -0,0 +1 @@
+| [...]/newtonsoft.json/13.0.3/lib/net6.0/Newtonsoft.Json.dll |

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/Assemblies.ql

@@ -0,0 +1,11 @@
+import csharp
+
+private string getPath(Assembly a) {
+  not a.getCompilation().getOutputAssembly() = a and
+  exists(string s | s = a.getFile().getAbsolutePath() |
+    result = "[...]/" + s.substring(s.indexOf("newtonsoft.json"), s.length())
+  )
+}
+
+from Assembly a
+select getPath(a)

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/CompilationInfo.expected

@@ -0,0 +1,16 @@
+| All Nuget feeds reachable | 0.0 |
+| Fallback nuget restore | 1.0 |
+| NuGet feed responsiveness checked | 1.0 |
+| Project files on filesystem | 1.0 |
+| Reachable fallback Nuget feed count | 2.0 |
+| Resolved assembly conflicts | 7.0 |
+| Resource extraction enabled | 0.0 |
+| Restored .NET framework variants | 0.0 |
+| Solution files on filesystem | 1.0 |
+| Source files generated | 0.0 |
+| Source files on filesystem | 1.0 |
+| Successfully ran fallback nuget restore | 1.0 |
+| Unresolved references | 0.0 |
+| UseWPF set | 0.0 |
+| UseWindowsForms set | 0.0 |
+| WebView extraction enabled | 1.0 |

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/CompilationInfo.ql

@@ -0,0 +1,15 @@
+import csharp
+import semmle.code.csharp.commons.Diagnostics
+
+query predicate compilationInfo(string key, float value) {
+  key != "Resolved references" and
+  not key.matches("Compiler diagnostic count for%") and
+  exists(Compilation c, string infoKey, string infoValue | infoValue = c.getInfo(infoKey) |
+    key = infoKey and
+    value = infoValue.toFloat()
+    or
+    not exists(infoValue.toFloat()) and
+    key = infoKey + ": " + infoValue and
+    value = 1
+  )
+}

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/diagnostics.expected

@@ -0,0 +1,42 @@
+{
+  "markdownMessage": "C# analysis with build-mode 'none' completed.",
+  "severity": "unknown",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/buildless/complete",
+    "name": "C# analysis with build-mode 'none' completed"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": false,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "C# was extracted with build-mode set to 'none'. This means that all C# source in the working directory will be scanned, with build tools, such as Nuget and Dotnet CLIs, only contributing information about external dependencies.",
+  "severity": "note",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/buildless/mode-active",
+    "name": "C# was extracted with build-mode set to 'none'"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
+  "severity": "warning",
+  "source": {
+    "extractorName": "csharp",
+    "id": "csharp/autobuilder/buildless/unreachable-feed",
+    "name": "Found unreachable Nuget feed in C# analysis with build-mode 'none'"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/proj/Program.cs

@@ -0,0 +1,6 @@
+class Program
+{
+    static void Main(string[] args)
+    {
+    }
+}

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/proj/nuget.config

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="x" value="https://www.nuget.org/api/v2/" />
+  </packageSources>
+</configuration>

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/proj/proj.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFrameworks>net8.0</TargetFrameworks>
+  </PropertyGroup>
+
+  <Target Name="DeleteBinObjFolders" BeforeTargets="Clean">
+    <RemoveDir Directories=".\bin" />
+    <RemoveDir Directories=".\obj" />
+  </Target>
+
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+  </ItemGroup>
+</Project>

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/standalone.sln

@@ -0,0 +1,19 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.5.002.0
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "proj", "proj\proj.csproj", "{6ED00460-7666-4AE9-A405-4B6C8B02279A}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {4ED55A1C-066C-43DF-B32E-7EAA035985EE}
+	EndGlobalSection
+EndGlobal

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/test.py

@@ -0,0 +1,14 @@
+from create_database_utils import *
+from diagnostics_test_utils import *
+import os
+
+# os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK"] = "true"        # Nuget feed check is enabled by default
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT"] = "1"     # 1ms, the GET request should fail with such short timeout
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT"] = "1"       # Limit the count of checks to 1
+
+# Making sure the reachability test succeeds when doing a fallback restore:
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_TIMEOUT"] = "1000"
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_LIMIT"] = "5"
+
+run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])
+check_diagnostics()

csharp/ql/lib/change-notes/2024-05-23-Version1.md

@@ -0,0 +1,4 @@
+---
+category: breaking
+---
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.

csharp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-all
-version: 0.10.2-dev
+version: 1.0.0-dev
 groups: csharp
 dbscheme: semmlecode.csharp.dbscheme
 extractor: csharp

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowDispatch.qll

@@ -24,19 +24,6 @@ newtype TReturnKind =
   TOutReturnKind(int i) { i = any(Parameter p | p.isOut()).getPosition() } or
   TRefReturnKind(int i) { i = any(Parameter p | p.isRef()).getPosition() }
 
-/**
- * A summarized callable where the summary should be used for dataflow analysis.
- */
-class DataFlowSummarizedCallable instanceof FlowSummary::SummarizedCallable {
-  DataFlowSummarizedCallable() {
-    not this.hasBody()
-    or
-    this.hasBody() and not this.applyGeneratedModel()
-  }
-
-  string toString() { result = super.toString() }
-}
-
 cached
 private module Cached {
   /**
@@ -47,7 +34,7 @@ private module Cached {
   cached
   newtype TDataFlowCallable =
     TCallable(Callable c) { c.isUnboundDeclaration() } or
-    TSummarizedCallable(DataFlowSummarizedCallable sc) or
+    TSummarizedCallable(FlowSummary::SummarizedCallable sc) or
     TFieldOrPropertyCallable(FieldOrProperty f) or
     TCapturedVariableCallable(LocalScopeVariable v) { v.isCaptured() }
 

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowPrivate.qll

@@ -1181,8 +1181,7 @@ private module Cached {
     or
     // Simple flow through library code is included in the exposed local
     // step relation, even though flow is technically inter-procedural
-    FlowSummaryImpl::Private::Steps::summaryThroughStepValue(nodeFrom, nodeTo,
+    FlowSummaryImpl::Private::Steps::summaryThroughStepValue(nodeFrom, nodeTo, _)
-      any(DataFlowSummarizedCallable sc))
   }
 
   cached

csharp/ql/lib/semmle/code/csharp/dataflow/internal/ExternalFlow.qll

@@ -554,7 +554,13 @@ private predicate interpretNeutral(UnboundCallable c, string kind, string proven
 
 // adapter class for converting Mad summaries to `SummarizedCallable`s
 private class SummarizedCallableAdapter extends SummarizedCallable {
-  SummarizedCallableAdapter() { interpretSummary(this, _, _, _, _, _) }
+  SummarizedCallableAdapter() {
+    exists(Provenance provenance | interpretSummary(this, _, _, _, provenance, _) |
+      not this.hasBody()
+      or
+      this.hasBody() and provenance.isManual()
+    )
+  }
 
   private predicate relevantSummaryElementManual(
     string input, string output, string kind, string model

csharp/ql/lib/semmle/code/csharp/dataflow/internal/TaintTrackingPrivate.qll

@@ -119,22 +119,19 @@ private module Cached {
     (
       // Simple flow through library code is included in the exposed local
       // step relation, even though flow is technically inter-procedural
-      FlowSummaryImpl::Private::Steps::summaryThroughStepTaint(nodeFrom, nodeTo,
+      FlowSummaryImpl::Private::Steps::summaryThroughStepTaint(nodeFrom, nodeTo, _)
-        any(DataFlowSummarizedCallable sc))
       or
       // Taint collection by adding a tainted element
       exists(DataFlow::ElementContent c |
         storeStep(nodeFrom, c, nodeTo)
         or
-        FlowSummaryImpl::Private::Steps::summarySetterStep(nodeFrom, c, nodeTo,
+        FlowSummaryImpl::Private::Steps::summarySetterStep(nodeFrom, c, nodeTo, _)
-          any(DataFlowSummarizedCallable sc))
       )
       or
       exists(DataFlow::Content c |
         readStep(nodeFrom, c, nodeTo)
         or
-        FlowSummaryImpl::Private::Steps::summaryGetterStep(nodeFrom, c, nodeTo,
+        FlowSummaryImpl::Private::Steps::summaryGetterStep(nodeFrom, c, nodeTo, _)
-          any(DataFlowSummarizedCallable sc))
       |
         // Taint members
         c = any(TaintedMember m).(FieldOrProperty).getContent()

csharp/ql/src/Language Abuse/ForeachCapture.ql

@@ -77,8 +77,7 @@ Element getAssignmentTarget(Expr e) {
 Element getCollectionAssignmentTarget(Expr e) {
   // Store into collection via method
   exists(DataFlowPrivate::PostUpdateNode postNode |
-    FlowSummaryImpl::Private::Steps::summarySetterStep(DataFlow::exprNode(e), _, postNode,
+    FlowSummaryImpl::Private::Steps::summarySetterStep(DataFlow::exprNode(e), _, postNode, _) and
-      any(DataFlowDispatch::DataFlowSummarizedCallable sc)) and
     result.(Variable).getAnAccess() = postNode.getPreUpdateNode().asExpr()
   )
   or

csharp/ql/src/change-notes/2024-05-23-Version1.md

@@ -0,0 +1,4 @@
+---
+category: breaking
+---
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.

csharp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-queries
-version: 0.8.17-dev
+version: 1.0.0-dev
 groups:
   - csharp
   - queries

csharp/ql/test/library-tests/dataflow/external-models/steps.ql

@@ -25,21 +25,17 @@ private class StepArgQualGenerated extends Method {
 query predicate summaryThroughStep(
   DataFlow::Node node1, DataFlow::Node node2, boolean preservesValue
 ) {
-  FlowSummaryImpl::Private::Steps::summaryThroughStepValue(node1, node2,
+  FlowSummaryImpl::Private::Steps::summaryThroughStepValue(node1, node2, _) and
-    any(DataFlowDispatch::DataFlowSummarizedCallable sc)) and
   preservesValue = true
   or
-  FlowSummaryImpl::Private::Steps::summaryThroughStepTaint(node1, node2,
+  FlowSummaryImpl::Private::Steps::summaryThroughStepTaint(node1, node2, _) and
-    any(DataFlowDispatch::DataFlowSummarizedCallable sc)) and
   preservesValue = false
 }
 
 query predicate summaryGetterStep(DataFlow::Node arg, DataFlow::Node out, Content c) {
-  FlowSummaryImpl::Private::Steps::summaryGetterStep(arg, c, out,
+  FlowSummaryImpl::Private::Steps::summaryGetterStep(arg, c, out, _)
-    any(DataFlowDispatch::DataFlowSummarizedCallable sc))
 }
 
 query predicate summarySetterStep(DataFlow::Node arg, DataFlow::Node out, Content c) {
-  FlowSummaryImpl::Private::Steps::summarySetterStep(arg, c, out,
+  FlowSummaryImpl::Private::Steps::summarySetterStep(arg, c, out, _)
-    any(DataFlowDispatch::DataFlowSummarizedCallable sc))
 }

docs/codeql/ql-language-reference/annotations.rst

@@ -446,7 +446,7 @@ The ``pragma[assume_small_delta]`` annotation has no effect and can be safely re
 Language pragmas
 ================
 
-**Available for**: |classes|, |characteristic predicates|, |member predicates|, |non-member predicates|
+**Available for**: |modules|, |classes|, |characteristic predicates|, |member predicates|, |non-member predicates|
 
 ``language[monotonicAggregates]``
 ---------------------------------

go/ql/consistency-queries/change-notes/2024-05-23-Version1.md

@@ -0,0 +1,4 @@
+---
+category: breaking
+---
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.

go/ql/consistency-queries/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql-go-consistency-queries
-version: 0.0.16-dev
+version: 1.0.0-dev
 groups:
   - go
   - queries

go/ql/lib/change-notes/2024-05-23-Version1.md

@@ -0,0 +1,4 @@
+---
+category: breaking
+---
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.

go/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/go-all
-version: 0.8.2-dev
+version: 1.0.0-dev
 groups: go
 dbscheme: go.dbscheme
 extractor: go

go/ql/src/change-notes/2024-05-23-Version1.md

@@ -0,0 +1,4 @@
+---
+category: breaking
+---
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.

go/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/go-queries
-version: 0.7.17-dev
+version: 1.0.0-dev
 groups:
   - go
   - queries

java/documentation/library-coverage/coverage.csv

@@ -87,7 +87,7 @@ java.rmi,,,71,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,71,
 java.security,21,,543,,,11,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,539,4
 java.sql,15,1,303,,,,1,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,,9,,,,,,,,,1,,,,303,
 java.text,,,134,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,134,
-java.time,,,476,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,388,88
+java.time,,,123,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,35,88
 java.util,47,2,1218,,,,,,,,,1,,,,,,,,,,,34,,,,2,,,,5,2,,1,2,,,,,,,,,,,,,2,,,704,514
 javafx.scene.web,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,
 javax.accessibility,,,31,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,31,

java/documentation/library-coverage/coverage.rst

@@ -18,10 +18,10 @@ Java framework & library support
    `Google Guava <https://guava.dev/>`_,``com.google.common.*``,,730,43,9,,,,,
    JBoss Logging,``org.jboss.logging``,,,324,,,,,,
    `JSON-java <https://github.com/stleary/JSON-java>`_,``org.json``,,236,,,,,,,
-   Java Standard Library,``java.*``,10,4620,240,80,,9,,,26
+   Java Standard Library,``java.*``,10,4267,240,80,,9,,,26
    Java extensions,"``javax.*``, ``jakarta.*``",69,3257,85,5,4,2,1,1,4
    Kotlin Standard Library,``kotlin*``,,1849,16,14,,,,,2
    `Spring <https://spring.io/>`_,``org.springframework.*``,38,481,122,5,,28,14,,35
    Others,"``actions.osgi``, ``antlr``, ``ch.ethz.ssh2``, ``cn.hutool.core.codec``, ``com.alibaba.druid.sql``, ``com.alibaba.fastjson2``, ``com.amazonaws.auth``, ``com.auth0.jwt.algorithms``, ``com.azure.identity``, ``com.esotericsoftware.kryo.io``, ``com.esotericsoftware.kryo5.io``, ``com.fasterxml.jackson.core``, ``com.fasterxml.jackson.databind``, ``com.google.gson``, ``com.hubspot.jinjava``, ``com.jcraft.jsch``, ``com.microsoft.sqlserver.jdbc``, ``com.mitchellbosecke.pebble``, ``com.mongodb``, ``com.opensymphony.xwork2``, ``com.rabbitmq.client``, ``com.sshtools.j2ssh.authentication``, ``com.sun.crypto.provider``, ``com.sun.jndi.ldap``, ``com.sun.net.httpserver``, ``com.sun.net.ssl``, ``com.sun.rowset``, ``com.sun.security.auth.module``, ``com.sun.security.ntlm``, ``com.sun.security.sasl.digest``, ``com.thoughtworks.xstream``, ``com.trilead.ssh2``, ``com.unboundid.ldap.sdk``, ``com.zaxxer.hikari``, ``flexjson``, ``freemarker.cache``, ``freemarker.template``, ``groovy.lang``, ``groovy.text``, ``groovy.util``, ``hudson``, ``io.jsonwebtoken``, ``io.netty.bootstrap``, ``io.netty.buffer``, ``io.netty.channel``, ``io.netty.handler.codec``, ``io.netty.handler.ssl``, ``io.netty.handler.stream``, ``io.netty.resolver``, ``io.netty.util``, ``javafx.scene.web``, ``jenkins``, ``jodd.json``, ``liquibase.database.jvm``, ``liquibase.statement.core``, ``net.schmizz.sshj``, ``net.sf.json``, ``net.sf.saxon.s9api``, ``ognl``, ``okhttp3``, ``org.acegisecurity``, ``org.antlr.runtime``, ``org.apache.commons.codec``, ``org.apache.commons.compress.archivers.tar``, ``org.apache.commons.exec``, ``org.apache.commons.httpclient.util``, ``org.apache.commons.jelly``, ``org.apache.commons.jexl2``, ``org.apache.commons.jexl3``, ``org.apache.commons.lang``, ``org.apache.commons.logging``, ``org.apache.commons.net``, ``org.apache.commons.ognl``, ``org.apache.cxf.catalog``, ``org.apache.cxf.common.classloader``, ``org.apache.cxf.common.jaxb``, ``org.apache.cxf.common.logging``, ``org.apache.cxf.configuration.jsse``, ``org.apache.cxf.helpers``, ``org.apache.cxf.resource``, ``org.apache.cxf.staxutils``, ``org.apache.cxf.tools.corba.utils``, ``org.apache.cxf.tools.util``, ``org.apache.cxf.transform``, ``org.apache.directory.ldap.client.api``, ``org.apache.hadoop.fs``, ``org.apache.hadoop.hive.metastore``, ``org.apache.hadoop.hive.ql.exec``, ``org.apache.hadoop.hive.ql.metadata``, ``org.apache.hc.client5.http.async.methods``, ``org.apache.hc.client5.http.classic.methods``, ``org.apache.hc.client5.http.fluent``, ``org.apache.hive.hcatalog.templeton``, ``org.apache.ibatis.jdbc``, ``org.apache.ibatis.mapping``, ``org.apache.log4j``, ``org.apache.shiro.codec``, ``org.apache.shiro.jndi``, ``org.apache.shiro.mgt``, ``org.apache.sshd.client.session``, ``org.apache.struts.beanvalidation.validation.interceptor``, ``org.apache.struts2``, ``org.apache.tools.ant``, ``org.apache.tools.zip``, ``org.apache.velocity.app``, ``org.apache.velocity.runtime``, ``org.codehaus.cargo.container.installer``, ``org.codehaus.groovy.control``, ``org.dom4j``, ``org.eclipse.jetty.client``, ``org.fusesource.leveldbjni``, ``org.geogebra.web.full.main``, ``org.gradle.api.file``, ``org.hibernate``, ``org.influxdb``, ``org.jdbi.v3.core``, ``org.jenkins.ui.icon``, ``org.jenkins.ui.symbol``, ``org.jooq``, ``org.keycloak.models.map.storage``, ``org.kohsuke.stapler``, ``org.mvel2``, ``org.openjdk.jmh.runner.options``, ``org.owasp.esapi``, ``org.pac4j.jwt.config.encryption``, ``org.pac4j.jwt.config.signature``, ``org.scijava.log``, ``org.slf4j``, ``org.thymeleaf``, ``org.w3c.dom``, ``org.xml.sax``, ``org.xmlpull.v1``, ``org.yaml.snakeyaml``, ``play.libs.ws``, ``play.mvc``, ``ratpack.core.form``, ``ratpack.core.handling``, ``ratpack.core.http``, ``ratpack.exec``, ``ratpack.form``, ``ratpack.func``, ``ratpack.handling``, ``ratpack.http``, ``ratpack.util``, ``retrofit2``, ``sun.awt``, ``sun.jvmstat.perfdata.monitor.protocol.local``, ``sun.jvmstat.perfdata.monitor.protocol.rmi``, ``sun.management.spi``, ``sun.misc``, ``sun.net.ftp``, ``sun.net.www.protocol.http``, ``sun.nio.ch``, ``sun.security.acl``, ``sun.security.jgss.krb5``, ``sun.security.krb5``, ``sun.security.pkcs``, ``sun.security.pkcs11``, ``sun.security.provider``, ``sun.security.ssl``, ``sun.security.x509``, ``sun.tools.jconsole``, ``sun.util.logging.internal``",131,10596,893,125,6,22,18,,208
-   Totals,,310,25483,2569,338,16,128,33,1,409
+   Totals,,310,25130,2569,338,16,128,33,1,409
 

java/ql/automodel/src/change-notes/2024-05-23-Version1.md

@@ -0,0 +1,4 @@
+---
+category: breaking
+---
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.

java/ql/automodel/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-automodel-queries
-version: 0.0.24-dev
+version: 1.0.0-dev
 groups:
     - java
     - automodel

java/ql/automodel/test/change-notes/2024-05-23-Version1.md

@@ -0,0 +1,4 @@
+---
+category: breaking
+---
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.

java/ql/automodel/test/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-automodel-tests
-version: 0.0.1-dev
+version: 1.0.0-dev
 groups:
     - java
     - automodel

java/ql/lib/change-notes/2024-05-15-static-field-side-effect.md

@@ -0,0 +1,4 @@
+---
+category: majorAnalysis
+---
+* Added support for data flow through side-effects on static fields. For example, when a static field containing an array is updated.

java/ql/lib/change-notes/2024-05-23-Version1.md

@@ -0,0 +1,4 @@
+---
+category: breaking
+---
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.

java/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-all
-version: 0.11.1-dev
+version: 1.0.0-dev
 groups: java
 dbscheme: config/semmlecode.dbscheme
 extractor: java

java/ql/lib/semmle/code/java/dataflow/ExternalFlow.qll

@@ -413,25 +413,28 @@ private string paramsStringQualified(Callable c) {
 }
 
 private Element interpretElement0(
-  string package, string type, boolean subtypes, string name, string signature
+  string package, string type, boolean subtypes, string name, string signature, boolean isExact
 ) {
   elementSpec(package, type, subtypes, name, signature, _) and
   (
     exists(Member m |
       (
-        result = m
+        result = m and isExact = true
         or
-        subtypes = true and result.(SrcMethod).overridesOrInstantiates+(m)
+        subtypes = true and result.(SrcMethod).overridesOrInstantiates+(m) and isExact = false
       ) and
       m.hasQualifiedName(package, type, name)
     |
-      signature = "" or
+      signature = ""
-      paramsStringQualified(m) = signature or
+      or
+      paramsStringQualified(m) = signature
+      or
       paramsString(m) = signature
     )
     or
     exists(RefType t |
       t.hasQualifiedName(package, type) and
+      isExact = false and
       (if subtypes = true then result.(SrcRefType).getASourceSupertype*() = t else result = t) and
       name = "" and
       signature = ""
@@ -442,13 +445,16 @@ private Element interpretElement0(
 /** Gets the source/sink/summary/neutral element corresponding to the supplied parameters. */
 cached
 Element interpretElement(
-  string package, string type, boolean subtypes, string name, string signature, string ext
+  string package, string type, boolean subtypes, string name, string signature, string ext,
+  boolean isExact
 ) {
   elementSpec(package, type, subtypes, name, signature, ext) and
-  exists(Element e | e = interpretElement0(package, type, subtypes, name, signature) |
+  exists(Element e, boolean isExact0 |
-    ext = "" and result = e
+    e = interpretElement0(package, type, subtypes, name, signature, isExact0)
+  |
+    ext = "" and result = e and isExact = isExact0
     or
-    ext = "Annotated" and result.(Annotatable).getAnAnnotation().getType() = e
+    ext = "Annotated" and result.(Annotatable).getAnAnnotation().getType() = e and isExact = false
   )
 }
 
@@ -538,13 +544,13 @@ predicate sinkNode(Node node, string kind) { sinkNode(node, kind, _) }
 
 // adapter class for converting Mad summaries to `SummarizedCallable`s
 private class SummarizedCallableAdapter extends SummarizedCallable {
-  SummarizedCallableAdapter() { summaryElement(this, _, _, _, _, _) }
+  SummarizedCallableAdapter() { summaryElement(this, _, _, _, _, _, _) }
 
   private predicate relevantSummaryElementManual(
     string input, string output, string kind, string model
   ) {
     exists(Provenance provenance |
-      summaryElement(this, input, output, kind, provenance, model) and
+      summaryElement(this, input, output, kind, provenance, model, _) and
       provenance.isManual()
     )
   }
@@ -553,11 +559,11 @@ private class SummarizedCallableAdapter extends SummarizedCallable {
     string input, string output, string kind, string model
   ) {
     exists(Provenance provenance |
-      summaryElement(this, input, output, kind, provenance, model) and
+      summaryElement(this, input, output, kind, provenance, model, _) and
       provenance.isGenerated()
     ) and
     not exists(Provenance provenance |
-      neutralElement(this, "summary", provenance) and
+      neutralElement(this, "summary", provenance, _) and
       provenance.isManual()
     )
   }
@@ -576,18 +582,23 @@ private class SummarizedCallableAdapter extends SummarizedCallable {
   }
 
   override predicate hasProvenance(Provenance provenance) {
-    summaryElement(this, _, _, _, provenance, _)
+    summaryElement(this, _, _, _, provenance, _, _)
   }
+
+  override predicate hasExactModel() { summaryElement(this, _, _, _, _, _, true) }
 }
 
 // adapter class for converting Mad neutrals to `NeutralCallable`s
 private class NeutralCallableAdapter extends NeutralCallable {
   string kind;
   string provenance_;
+  boolean exact;
 
-  NeutralCallableAdapter() { neutralElement(this, kind, provenance_) }
+  NeutralCallableAdapter() { neutralElement(this, kind, provenance_, exact) }
 
   override string getKind() { result = kind }
 
   override predicate hasProvenance(Provenance provenance) { provenance = provenance_ }
+
+  override predicate hasExactModel() { exact = true }
 }

java/ql/lib/semmle/code/java/dataflow/FlowSummary.qll

@@ -135,6 +135,8 @@ private class SummarizedSyntheticCallableAdapter extends SummarizedCallable, TSy
       model = sc
     )
   }
+
+  override predicate hasExactModel() { any() }
 }
 
 deprecated class RequiredSummaryComponentStack = Impl::Private::RequiredSummaryComponentStack;

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowDispatch.qll

@@ -19,7 +19,21 @@ private module DispatchImpl {
     )
   }
 
+  private predicate hasExactManualModel(Call c, Callable tgt) {
+    tgt = c.getCallee().getSourceDeclaration() and
+    (
+      exists(Impl::Public::SummarizedCallable sc |
+        sc.getACall() = c and sc.hasExactModel() and sc.hasManualModel()
+      )
+      or
+      exists(Impl::Public::NeutralSummaryCallable nc |
+        nc.getACall() = c and nc.hasExactModel() and nc.hasManualModel()
+      )
+    )
+  }
+
   private Callable sourceDispatch(Call c) {
+    not hasExactManualModel(c, result) and
     result = VirtualDispatch::viableCallable(c) and
     if VirtualDispatch::lowConfidenceDispatchTarget(c, result)
     then not hasHighConfidenceTarget(c)
@@ -122,12 +136,18 @@ private module DispatchImpl {
     mayBenefitFromCallContext(call.asCall(), _, _)
   }
 
+  bindingset[call, tgt]
+  pragma[inline_late]
+  private predicate viableCallableFilter(DataFlowCall call, DataFlowCallable tgt) {
+    tgt = viableCallable(call)
+  }
+
   /**
    * Gets a viable dispatch target of `call` in the context `ctx`. This is
    * restricted to those `call`s for which a context might make a difference.
    */
   DataFlowCallable viableImplInCallContext(DataFlowCall call, DataFlowCall ctx) {
-    result = viableCallable(call) and
+    viableCallableFilter(call, result) and
     exists(int i, Callable c, Method def, RefType t, boolean exact, MethodCall ma |
       ma = call.asCall() and
       mayBenefitFromCallContext(ma, c, i) and

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowPrivate.qll

@@ -40,8 +40,11 @@ private predicate fieldStep(Node node1, Node node2) {
   exists(Field f |
     // Taint fields through assigned values only if they're static
     f.isStatic() and
-    f.getAnAssignedValue() = node1.asExpr() and
     node2.(FieldValueNode).getField() = f
+  |
+    f.getAnAssignedValue() = node1.asExpr()
+    or
+    f.getAnAccess() = node1.(PostUpdateNode).getPreUpdateNode().asExpr()
   )
   or
   exists(Field f, FieldRead fr |

java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll

@@ -131,7 +131,7 @@ private predicate relatedArgSpec(Callable c, string spec) {
     sourceModel(namespace, type, subtypes, name, signature, ext, spec, _, _, _) or
     sinkModel(namespace, type, subtypes, name, signature, ext, spec, _, _, _)
   |
-    c = interpretElement(namespace, type, subtypes, name, signature, ext)
+    c = interpretElement(namespace, type, subtypes, name, signature, ext, _)
   )
 }
 
@@ -202,7 +202,7 @@ module SourceSinkInterpretationInput implements
       sourceModel(namespace, type, subtypes, name, signature, ext, originalOutput, kind, provenance,
         madId) and
       model = "MaD:" + madId.toString() and
-      baseSource = interpretElement(namespace, type, subtypes, name, signature, ext) and
+      baseSource = interpretElement(namespace, type, subtypes, name, signature, ext, _) and
       (
         e = baseSource and output = originalOutput
         or
@@ -221,7 +221,7 @@ module SourceSinkInterpretationInput implements
       sinkModel(namespace, type, subtypes, name, signature, ext, originalInput, kind, provenance,
         madId) and
       model = "MaD:" + madId.toString() and
-      baseSink = interpretElement(namespace, type, subtypes, name, signature, ext) and
+      baseSink = interpretElement(namespace, type, subtypes, name, signature, ext, _) and
       (
         e = baseSink and originalInput = input
         or
@@ -310,7 +310,7 @@ module Private {
      */
     predicate summaryElement(
       Input::SummarizedCallableBase c, string input, string output, string kind, string provenance,
-      string model
+      string model, boolean isExact
     ) {
       exists(
         string namespace, string type, boolean subtypes, string name, string signature, string ext,
@@ -320,7 +320,7 @@ module Private {
         summaryModel(namespace, type, subtypes, name, signature, ext, originalInput, originalOutput,
           kind, provenance, madId) and
         model = "MaD:" + madId.toString() and
-        baseCallable = interpretElement(namespace, type, subtypes, name, signature, ext) and
+        baseCallable = interpretElement(namespace, type, subtypes, name, signature, ext, isExact) and
         (
           c.asCallable() = baseCallable and input = originalInput and output = originalOutput
           or
@@ -336,10 +336,12 @@ module Private {
      * Holds if a neutral model exists for `c` of kind `kind`
      * and with provenance `provenance`.
      */
-    predicate neutralElement(Input::SummarizedCallableBase c, string kind, string provenance) {
+    predicate neutralElement(
+      Input::SummarizedCallableBase c, string kind, string provenance, boolean isExact
+    ) {
       exists(string namespace, string type, string name, string signature |
         neutralModel(namespace, type, name, signature, kind, provenance) and
-        c.asCallable() = interpretElement(namespace, type, false, name, signature, "")
+        c.asCallable() = interpretElement(namespace, type, false, name, signature, "", isExact)
       )
     }
   }

java/ql/src/Security/CWE/CWE-501/TrustBoundaryViolation.ql

@@ -16,5 +16,5 @@ import TrustBoundaryFlow::PathGraph
 
 from TrustBoundaryFlow::PathNode source, TrustBoundaryFlow::PathNode sink
 where TrustBoundaryFlow::flowPath(source, sink)
-select sink.getNode(), sink, source,
+select sink.getNode(), source, sink,
-  "This servlet reads data from a remote source and writes it to a session variable."
+  "This servlet reads data from a $@ and writes it to a session variable.", source, "remote source"

java/ql/src/change-notes/2024-05-23-Version1.md

@@ -0,0 +1,4 @@
+---
+category: breaking
+---
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.

java/ql/src/change-notes/2024-05-23-trusted-boundary-violation.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The alert message for the query "Trust boundary violation" (`java/trust-boundary-violation`) has been updated to include a link to the remote source.

java/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-queries
-version: 0.8.17-dev
+version: 1.0.0-dev
 groups:
   - java
   - queries

java/ql/src/utils/modeleditor/ModelEditor.qll

@@ -77,7 +77,7 @@ class Endpoint extends Callable {
   predicate isNeutral() {
     exists(string namespace, string type, string name, string signature |
       neutralModel(namespace, type, name, signature, _, _) and
-      this = interpretElement(namespace, type, false, name, signature, "")
+      this = interpretElement(namespace, type, false, name, signature, "", _)
     )
   }
 

java/ql/src/utils/modelgenerator/internal/CaptureModelsSpecific.qll

@@ -80,10 +80,11 @@ predicate isUninterestingForDataFlowModels(Callable api) {
 predicate isUninterestingForTypeBasedFlowModels(Callable api) { none() }
 
 /**
- * A class of Callables that are relevant for generating summary, source and sinks models for.
+ * A class of callables that are potentially relevant for generating summary, source, sink
+ * and neutral models.
  *
- * In the Standard library and 3rd party libraries it the Callables that can be called
+ * In the Standard library and 3rd party libraries it is the callables (or callables that have a
- * from outside the library itself.
+ * super implementation) that can be called from outside the library itself.
  */
 class TargetApiSpecific extends Callable {
   private Callable lift;
@@ -97,6 +98,11 @@ class TargetApiSpecific extends Callable {
    * Gets the callable that a model will be lifted to.
    */
   Callable lift() { result = lift }
+
+  /**
+   * Holds if this callable is relevant in terms of generating models.
+   */
+  predicate isRelevant() { relevant(this) }
 }
 
 private string isExtensible(Callable c) {
@@ -114,15 +120,13 @@ private string typeAsModel(Callable c) {
   )
 }
 
-private predicate partialLiftedModel(
+private predicate partialModel(
-  TargetApiSpecific api, string type, string extensible, string name, string parameters
+  Callable api, string type, string extensible, string name, string parameters
 ) {
-  exists(Callable c | c = api.lift() |
+  type = typeAsModel(api) and
-    type = typeAsModel(c) and
+  extensible = isExtensible(api) and
-    extensible = isExtensible(c) and
+  name = api.getName() and
-    name = c.getName() and
+  parameters = ExternalFlow::paramsString(api)
-    parameters = ExternalFlow::paramsString(c)
-  )
 }
 
 /**
@@ -130,7 +134,7 @@ private predicate partialLiftedModel(
  */
 string asPartialModel(TargetApiSpecific api) {
   exists(string type, string extensible, string name, string parameters |
-    partialLiftedModel(api, type, extensible, name, parameters) and
+    partialModel(api.lift(), type, extensible, name, parameters) and
     result =
       type + ";" //
         + extensible + ";" //
@@ -145,7 +149,7 @@ string asPartialModel(TargetApiSpecific api) {
  */
 string asPartialNeutralModel(TargetApiSpecific api) {
   exists(string type, string name, string parameters |
-    partialLiftedModel(api, type, _, name, parameters) and
+    partialModel(api, type, _, name, parameters) and
     result =
       type + ";" //
         + name + ";" //

java/ql/src/utils/modelgenerator/internal/CaptureSummaryFlowQuery.qll

@@ -79,5 +79,6 @@ string captureFlow(DataFlowTargetApi api) {
  */
 string captureNoFlow(DataFlowTargetApi api) {
   not exists(DataFlowTargetApi api0 | exists(captureFlow(api0)) and api0.lift() = api.lift()) and
+  api.isRelevant() and
   result = ModelPrinting::asNeutralSummaryModel(api)
 }

java/ql/test/library-tests/dataflow/fields/G.java

@@ -0,0 +1,21 @@
+public class G {
+  static Object[] f;
+
+  void sink(Object o) { }
+
+  void runsink() {
+    sink(f[0]);
+  }
+
+  void test1() {
+    f[0] = new Object();
+  }
+
+  void test2() {
+    addObj(f);
+  }
+
+  void addObj(Object[] xs) {
+    xs[0] = new Object();
+  }
+}

java/ql/test/library-tests/dataflow/fields/flow.expected

@@ -29,3 +29,5 @@
 | F.java:5:14:5:25 | new Object(...) | F.java:20:10:20:17 | f.Field1 |
 | F.java:10:16:10:27 | new Object(...) | F.java:15:10:15:17 | f.Field1 |
 | F.java:24:9:24:20 | new Object(...) | F.java:33:10:33:17 | f.Field1 |
+| G.java:11:12:11:23 | new Object(...) | G.java:7:10:7:13 | ...[...] |
+| G.java:19:13:19:24 | new Object(...) | G.java:7:10:7:13 | ...[...] |

java/ql/test/utils/modelgenerator/dataflow/p/ImplOfExternalSPI.java

@@ -7,7 +7,7 @@ import java.nio.file.Files;
 public class ImplOfExternalSPI extends AbstractImplOfExternalSPI {
 
   // sink=p;AbstractImplOfExternalSPI;true;accept;(File);;Argument[0];path-injection;df-generated
-  // neutral=p;AbstractImplOfExternalSPI;accept;(File);summary;df-generated
+  // neutral=p;ImplOfExternalSPI;accept;(File);summary;df-generated
   @Override
   public boolean accept(File pathname) {
     try {

java/ql/test/utils/modelgenerator/dataflow/p/Inheritance.java

@@ -88,4 +88,28 @@ public class Inheritance {
       return s;
     }
   }
+
+  public interface INeutral {
+    String id(String s);
+  }
+
+  public class F implements INeutral {
+    // neutral=p;Inheritance$F;id;(String);summary;df-generated
+    public String id(String s) {
+      return "";
+    }
+  }
+
+  public class G implements INeutral {
+    // neutral=p;Inheritance$G;id;(String);summary;df-generated
+    public String id(String s) {
+      return "";
+    }
+  }
+
+  private class H implements INeutral {
+    public String id(String s) {
+      return "";
+    }
+  }
 }

java/ql/test/utils/modelgenerator/dataflow/p/PrivateFlowViaPublicInterface.java

@@ -45,7 +45,6 @@ public class PrivateFlowViaPublicInterface {
       return null;
     }
 
-    // neutral=p;PrivateFlowViaPublicInterface$SPI;openStreamNone;();summary;df-generated
     @Override
     public OutputStream openStreamNone() throws IOException {
       return new FileOutputStream(new RandomPojo().someFile);

javascript/ql/lib/change-notes/2024-05-23-Version1.md

@@ -0,0 +1,4 @@
+---
+category: breaking
+---
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.

javascript/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/javascript-all
-version: 0.9.2-dev
+version: 1.0.0-dev
 groups: javascript
 dbscheme: semmlecode.javascript.dbscheme
 extractor: javascript

javascript/ql/src/change-notes/2024-05-23-Version1.md

@@ -0,0 +1,4 @@
+---
+category: breaking
+---
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.

javascript/ql/src/experimental/semmle/javascript/Execa.qll

@@ -0,0 +1,211 @@
+/**
+ * Models the `execa` library in terms of `FileSystemAccess` and `SystemCommandExecution`.
+ */
+
+import javascript
+
+/**
+ * Provide model for [Execa](https://github.com/sindresorhus/execa) package
+ */
+module Execa {
+  /**
+   * The Execa input file read and output file write
+   */
+  class ExecaFileSystemAccess extends FileSystemReadAccess, DataFlow::Node {
+    API::Node execaArg;
+    boolean isPipedToFile;
+
+    ExecaFileSystemAccess() {
+      (
+        execaArg = API::moduleImport("execa").getMember("$").getParameter(0) and
+        isPipedToFile = false
+        or
+        execaArg =
+          API::moduleImport("execa")
+              .getMember(["execa", "execaCommand", "execaCommandSync", "execaSync"])
+              .getParameter([0, 1, 2]) and
+        isPipedToFile = false
+        or
+        execaArg =
+          API::moduleImport("execa")
+              .getMember(["execa", "execaCommand", "execaCommandSync", "execaSync"])
+              .getReturn()
+              .getMember(["pipeStdout", "pipeAll", "pipeStderr"])
+              .getParameter(0) and
+        isPipedToFile = true
+      ) and
+      this = execaArg.asSink()
+    }
+
+    override DataFlow::Node getADataNode() { none() }
+
+    override DataFlow::Node getAPathArgument() {
+      result = execaArg.getMember("inputFile").asSink() and isPipedToFile = false
+      or
+      result = execaArg.asSink() and isPipedToFile = true
+    }
+  }
+
+  /**
+   * A call to `execa.execa` or `execa.execaSync`
+   */
+  class ExecaCall extends API::CallNode {
+    boolean isSync;
+
+    ExecaCall() {
+      this = API::moduleImport("execa").getMember("execa").getACall() and
+      isSync = false
+      or
+      this = API::moduleImport("execa").getMember("execaSync").getACall() and
+      isSync = true
+    }
+  }
+
+  /**
+   * The system command execution nodes for `execa.execa` or `execa.execaSync` functions
+   */
+  class ExecaExec extends SystemCommandExecution, ExecaCall {
+    ExecaExec() { isSync = [false, true] }
+
+    override DataFlow::Node getACommandArgument() { result = this.getArgument(0) }
+
+    override predicate isShellInterpreted(DataFlow::Node arg) {
+      // if shell: true then first and second args are sinks
+      // options can be third argument
+      arg = [this.getArgument(0), this.getParameter(1).getUnknownMember().asSink()] and
+      isExecaShellEnable(this.getParameter(2))
+      or
+      // options can be second argument
+      arg = this.getArgument(0) and
+      isExecaShellEnable(this.getParameter(1))
+    }
+
+    override DataFlow::Node getArgumentList() {
+      // execa(cmd, [arg]);
+      exists(DataFlow::Node arg | arg = this.getArgument(1) |
+        // if it is a object then it is a option argument not command argument
+        result = arg and not arg.asExpr() instanceof ObjectExpr
+      )
+    }
+
+    override predicate isSync() { isSync = true }
+
+    override DataFlow::Node getOptionsArg() {
+      result = this.getLastArgument() and result.asExpr() instanceof ObjectExpr
+    }
+  }
+
+  /**
+   * A call to `execa.$` or `execa.$.sync` or `execa.$({})` or `execa.$.sync({})` tag functions
+   */
+  private class ExecaScriptCall extends API::CallNode {
+    boolean isSync;
+
+    ExecaScriptCall() {
+      exists(API::Node script |
+        script =
+          [
+            API::moduleImport("execa").getMember("$"),
+            API::moduleImport("execa").getMember("$").getReturn()
+          ]
+      |
+        this = script.getACall() and
+        isSync = false
+        or
+        this = script.getMember("sync").getACall() and
+        isSync = true
+      )
+    }
+  }
+
+  /**
+   * The system command execution nodes for `execa.$` or `execa.$.sync` tag functions
+   */
+  class ExecaScript extends SystemCommandExecution, ExecaScriptCall {
+    ExecaScript() { isSync = [false, true] }
+
+    override DataFlow::Node getACommandArgument() {
+      result = this.getParameter(1).asSink() and
+      not isTaggedTemplateFirstChildAnElement(this.getParameter(1).asSink().asExpr().getParent())
+    }
+
+    override predicate isShellInterpreted(DataFlow::Node arg) {
+      isExecaShellEnable(this.getParameter(0)) and
+      arg = this.getAParameter().asSink()
+    }
+
+    override DataFlow::Node getArgumentList() {
+      result = this.getParameter(any(int i | i >= 1)).asSink() and
+      isTaggedTemplateFirstChildAnElement(this.getParameter(1).asSink().asExpr().getParent())
+      or
+      result = this.getParameter(any(int i | i >= 2)).asSink() and
+      not isTaggedTemplateFirstChildAnElement(this.getParameter(1).asSink().asExpr().getParent())
+    }
+
+    override DataFlow::Node getOptionsArg() { result = this.getParameter(0).asSink() }
+
+    override predicate isSync() { isSync = true }
+  }
+
+  /**
+   * A call to `execa.execaCommandSync` or `execa.execaCommand`
+   */
+  private class ExecaCommandCall extends API::CallNode {
+    boolean isSync;
+
+    ExecaCommandCall() {
+      this = API::moduleImport("execa").getMember("execaCommandSync").getACall() and
+      isSync = true
+      or
+      this = API::moduleImport("execa").getMember("execaCommand").getACall() and
+      isSync = false
+    }
+  }
+
+  /**
+   * The system command execution nodes for `execa.execaCommand` or `execa.execaCommandSync` functions
+   */
+  class ExecaCommandExec extends SystemCommandExecution, ExecaCommandCall {
+    ExecaCommandExec() { isSync = [false, true] }
+
+    override DataFlow::Node getACommandArgument() {
+      result = this.(DataFlow::CallNode).getArgument(0)
+    }
+
+    override DataFlow::Node getArgumentList() {
+      // execaCommand(`${cmd} ${arg}`);
+      result.asExpr() = this.getParameter(0).asSink().asExpr().getAChildExpr() and
+      not result.asExpr() = this.getArgument(0).asExpr().getChildExpr(0)
+    }
+
+    override predicate isShellInterpreted(DataFlow::Node arg) {
+      // execaCommandSync(`${cmd} ${arg}`, {shell: true})
+      arg.asExpr() = this.getArgument(0).asExpr().getAChildExpr+() and
+      isExecaShellEnable(this.getParameter(1))
+      or
+      // there is only one argument that is constructed in previous nodes,
+      // it makes sanitizing really hard to select whether it is vulnerable to argument injection or not
+      arg = this.getParameter(0).asSink() and
+      not exists(this.getArgument(0).asExpr().getChildExpr(1))
+    }
+
+    override predicate isSync() { isSync = true }
+
+    override DataFlow::Node getOptionsArg() {
+      result = this.getLastArgument() and result.asExpr() instanceof ObjectExpr
+    }
+  }
+
+  /** Gets a TemplateLiteral and check if first child is a template element */
+  private predicate isTaggedTemplateFirstChildAnElement(TemplateLiteral templateLit) {
+    exists(templateLit.getChildExpr(0).(TemplateElement))
+  }
+
+  /**
+   * Holds whether Execa has shell enabled options or not, get Parameter responsible for options
+   */
+  pragma[inline]
+  private predicate isExecaShellEnable(API::Node n) {
+    n.getMember("shell").asSink().asExpr().(BooleanLiteral).getValue() = "true"
+  }
+}

javascript/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/javascript-queries
-version: 0.8.17-dev
+version: 1.0.0-dev
 groups:
   - javascript
   - queries

javascript/ql/test/experimental/Execa/CommandInjection/tests.expected

@@ -0,0 +1,22 @@
+passingPositiveTests
+| PASSED | CommandInjection | tests.js:11:46:11:70 | // test ... jection |
+| PASSED | CommandInjection | tests.js:12:43:12:67 | // test ... jection |
+| PASSED | CommandInjection | tests.js:13:63:13:87 | // test ... jection |
+| PASSED | CommandInjection | tests.js:14:62:14:86 | // test ... jection |
+| PASSED | CommandInjection | tests.js:15:60:15:84 | // test ... jection |
+| PASSED | CommandInjection | tests.js:17:45:17:69 | // test ... jection |
+| PASSED | CommandInjection | tests.js:18:42:18:66 | // test ... jection |
+| PASSED | CommandInjection | tests.js:19:62:19:86 | // test ... jection |
+| PASSED | CommandInjection | tests.js:20:63:20:87 | // test ... jection |
+| PASSED | CommandInjection | tests.js:21:60:21:84 | // test ... jection |
+| PASSED | CommandInjection | tests.js:23:43:23:67 | // test ... jection |
+| PASSED | CommandInjection | tests.js:24:40:24:64 | // test ... jection |
+| PASSED | CommandInjection | tests.js:25:40:25:64 | // test ... jection |
+| PASSED | CommandInjection | tests.js:26:60:26:84 | // test ... jection |
+| PASSED | CommandInjection | tests.js:28:41:28:65 | // test ... jection |
+| PASSED | CommandInjection | tests.js:29:58:29:82 | // test ... jection |
+| PASSED | CommandInjection | tests.js:31:51:31:75 | // test ... jection |
+| PASSED | CommandInjection | tests.js:32:68:32:92 | // test ... jection |
+| PASSED | CommandInjection | tests.js:34:49:34:73 | // test ... jection |
+| PASSED | CommandInjection | tests.js:35:66:35:90 | // test ... jection |
+failingPositiveTests

javascript/ql/test/experimental/Execa/CommandInjection/tests.js

@@ -0,0 +1,36 @@
+import { execa, execaSync, execaCommand, execaCommandSync, $ } from 'execa';
+import http from 'node:http'
+import url from 'url'
+
+http.createServer(async function (req, res) {
+    let cmd = url.parse(req.url, true).query["cmd"][0];
+    let arg1 = url.parse(req.url, true).query["arg1"];
+    let arg2 = url.parse(req.url, true).query["arg2"];
+    let arg3 = url.parse(req.url, true).query["arg3"];
+
+    await $`${cmd} ${arg1} ${arg2} ${arg3}`; // test: CommandInjection
+    await $`ssh ${arg1} ${arg2} ${arg3}`; // test: CommandInjection
+    $({ shell: false }).sync`${cmd} ${arg1} ${arg2} ${arg3}`; // test: CommandInjection
+    $({ shell: true }).sync`${cmd} ${arg1} ${arg2} ${arg3}`; // test: CommandInjection
+    $({ shell: false }).sync`ssh ${arg1} ${arg2} ${arg3}`; // test: CommandInjection
+
+    $.sync`${cmd} ${arg1} ${arg2} ${arg3}`; // test: CommandInjection
+    $.sync`ssh ${arg1} ${arg2} ${arg3}`; // test: CommandInjection
+    await $({ shell: true })`${cmd} ${arg1} ${arg2} ${arg3}` // test: CommandInjection
+    await $({ shell: false })`${cmd} ${arg1} ${arg2} ${arg3}` // test: CommandInjection
+    await $({ shell: false })`ssh ${arg1} ${arg2} ${arg3}` // test: CommandInjection
+
+    await execa(cmd, [arg1, arg2, arg3]); // test: CommandInjection
+    await execa(cmd, { shell: true }); // test: CommandInjection
+    await execa(cmd, { shell: true }); // test: CommandInjection
+    await execa(cmd, [arg1, arg2, arg3], { shell: true }); // test: CommandInjection
+
+    execaSync(cmd, [arg1, arg2, arg3]); // test: CommandInjection
+    execaSync(cmd, [arg1, arg2, arg3], { shell: true }); // test: CommandInjection
+
+    await execaCommand(cmd + arg1 + arg2 + arg3); // test: CommandInjection
+    await execaCommand(cmd + arg1 + arg2 + arg3, { shell: true }); // test: CommandInjection
+
+    execaCommandSync(cmd + arg1 + arg2 + arg3); // test: CommandInjection
+    execaCommandSync(cmd + arg1 + arg2 + arg3, { shell: true }); // test: CommandInjection
+});

javascript/ql/test/experimental/Execa/CommandInjection/tests.ql

@@ -0,0 +1,38 @@
+import javascript
+
+class InlineTest extends LineComment {
+  string tests;
+
+  InlineTest() { tests = this.getText().regexpCapture("\\s*test:(.*)", 1) }
+
+  string getPositiveTest() {
+    result = tests.trim().splitAt(",").trim() and not result.matches("!%")
+  }
+
+  predicate hasPositiveTest(string test) { test = this.getPositiveTest() }
+
+  predicate inNode(DataFlow::Node n) {
+    this.getLocation().getFile() = n.getFile() and
+    this.getLocation().getStartLine() = n.getStartLine()
+  }
+}
+
+import experimental.semmle.javascript.Execa
+
+query predicate passingPositiveTests(string res, string expectation, InlineTest t) {
+  res = "PASSED" and
+  t.hasPositiveTest(expectation) and
+  expectation = "CommandInjection" and
+  exists(SystemCommandExecution n |
+    t.inNode(n.getArgumentList()) or t.inNode(n.getACommandArgument())
+  )
+}
+
+query predicate failingPositiveTests(string res, string expectation, InlineTest t) {
+  res = "FAILED" and
+  t.hasPositiveTest(expectation) and
+  expectation = "CommandInjection" and
+  not exists(SystemCommandExecution n |
+    t.inNode(n.getArgumentList()) or t.inNode(n.getACommandArgument())
+  )
+}

javascript/ql/test/experimental/Execa/PathInjection/tests.expected

@@ -0,0 +1,6 @@
+passingPositiveTests
+| PASSED | PathInjection | tests.js:9:43:9:64 | // test ... jection |
+| PASSED | PathInjection | tests.js:12:50:12:71 | // test ... jection |
+| PASSED | PathInjection | tests.js:15:61:15:82 | // test ... jection |
+| PASSED | PathInjection | tests.js:18:73:18:94 | // test ... jection |
+failingPositiveTests

javascript/ql/test/experimental/Execa/PathInjection/tests.js

@@ -0,0 +1,19 @@
+import { execa, $ } from 'execa';
+import http from 'node:http'
+import url from 'url'
+
+http.createServer(async function (req, res) {
+    let filePath = url.parse(req.url, true).query["filePath"][0];
+
+    // Piping to stdin from a file
+    await $({ inputFile: filePath })`cat` // test: PathInjection
+
+    // Piping to stdin from a file
+    await execa('cat', { inputFile: filePath }); // test: PathInjection
+
+    // Piping Stdout to file
+    await execa('echo', ['example3']).pipeStdout(filePath); // test: PathInjection
+
+    // Piping all of command output to file
+    await execa('echo', ['example4'], { all: true }).pipeAll(filePath); // test: PathInjection
+});

javascript/ql/test/experimental/Execa/PathInjection/tests.ql

@@ -0,0 +1,34 @@
+import javascript
+
+class InlineTest extends LineComment {
+  string tests;
+
+  InlineTest() { tests = this.getText().regexpCapture("\\s*test:(.*)", 1) }
+
+  string getPositiveTest() {
+    result = tests.trim().splitAt(",").trim() and not result.matches("!%")
+  }
+
+  predicate hasPositiveTest(string test) { test = this.getPositiveTest() }
+
+  predicate inNode(DataFlow::Node n) {
+    this.getLocation().getFile() = n.getFile() and
+    this.getLocation().getStartLine() = n.getStartLine()
+  }
+}
+
+import experimental.semmle.javascript.Execa
+
+query predicate passingPositiveTests(string res, string expectation, InlineTest t) {
+  res = "PASSED" and
+  t.hasPositiveTest(expectation) and
+  expectation = "PathInjection" and
+  exists(FileSystemReadAccess n | t.inNode(n.getAPathArgument()))
+}
+
+query predicate failingPositiveTests(string res, string expectation, InlineTest t) {
+  res = "FAILED" and
+  t.hasPositiveTest(expectation) and
+  expectation = "PathInjection" and
+  not exists(FileSystemReadAccess n | t.inNode(n.getAPathArgument()))
+}

misc/suite-helpers/change-notes/2024-05-23-Version1.md

@@ -0,0 +1,4 @@
+---
+category: breaking
+---
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.

misc/suite-helpers/qlpack.yml

@@ -1,4 +1,4 @@
 name: codeql/suite-helpers
-version: 0.7.17-dev
+version: 1.0.0-dev
 groups: shared
 warnOnImplicitThis: true

python/extractor/tsg-python/Cargo.Bazel.lock

@@ -1,5 +1,5 @@
 {
-  "checksum": "35a1ce4b6c4f997c496c11d3a8fcfaadc5833dfd41bebb022941687d73dde159",
+  "checksum": "14572337bc5747880ff328af42451cce3549f743dc79eac7314f3b3f55b74d25",
   "crates": {
     "ahash 0.4.7": {
       "name": "ahash",
@@ -1755,7 +1755,7 @@
           ],
           "selects": {}
         },
-        "edition": "2018",
+        "edition": "2021",
         "version": "0.1.0"
       },
       "license": null,
@@ -1986,7 +1986,6 @@
         "crate_features": {
           "common": [
             "consoleapi",
-            "errhandlingapi",
             "minwinbase",
             "minwindef",
             "processenv",

python/extractor/tsg-python/Cargo.toml

@@ -4,7 +4,7 @@
 name = "tsg-python"
 version = "0.1.0"
 authors = ["Taus Brock-Nannestad <tausbn@github.com>"]
-edition = "2018"
+edition = "2021"
 
 # When changing/updating these, the `Cargo.Bazel.lock` file has to be regenerated.
 # Run `CARGO_BAZEL_REPIN=true CARGO_BAZEL_REPIN_ONLY=py_deps ./tools/bazel sync --only=py_deps`

python/extractor/tsg-python/rust-toolchain.toml

@@ -2,6 +2,6 @@
 # extractor. It is set to the lowest version of Rust we want to support.
 
 [toolchain]
-channel = "1.68"
+channel = "1.74"
 profile = "minimal"
 components = [ "rustfmt" ]

python/ql/lib/change-notes/2024-05-23-Version1.md

@@ -0,0 +1,4 @@
+---
+category: breaking
+---
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.

python/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/python-all
-version: 0.12.2-dev
+version: 1.0.0-dev
 groups: python
 dbscheme: semmlecode.python.dbscheme
 extractor: python

python/ql/src/change-notes/2024-05-23-Version1.md

@@ -0,0 +1,4 @@
+---
+category: breaking
+---
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.

python/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/python-queries
-version: 0.9.17-dev
+version: 1.0.0-dev
 groups:
   - python
   - queries

ql/extractor/src/extractor.rs

@@ -20,12 +20,7 @@ pub struct Options {
 }
 
 pub fn run(options: Options) -> std::io::Result<()> {
-    tracing_subscriber::fmt()
+    codeql_extractor::extractor::set_tracing_level("ql");
-        .with_target(false)
-        .without_time()
-        .with_level(true)
-        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
-        .init();
 
     let extractor = simple::Extractor {
         prefix: "ql".to_string(),

ql/extractor/src/generator.rs

@@ -15,12 +15,7 @@ pub struct Options {
 }
 
 pub fn run(options: Options) -> std::io::Result<()> {
-    tracing_subscriber::fmt()
+    codeql_extractor::extractor::set_tracing_level("ql");
-        .with_target(false)
-        .without_time()
-        .with_level(true)
-        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
-        .init();
 
     let languages = vec![
         Language {

ql/ql/src/change-notes/2024-05-23-Version1.md

@@ -0,0 +1,4 @@
+---
+category: breaking
+---
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.

ql/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/ql
-version: 0.1.0-dev
+version: 1.0.0-dev
 groups:
   - ql
   - queries

ql/rust-toolchain.toml

@@ -2,6 +2,6 @@
 # extractor. It is set to the lowest version of Rust we want to support.
 
 [toolchain]
-channel = "1.70"
+channel = "1.74"
 profile = "minimal"
 components = [ "rustfmt" ]

ruby/extractor/Cargo.toml

@@ -34,7 +34,7 @@ lazy_static = "1.4.0"
 # of lock-file update time, but `rules_rust` pins generates a bazel rule that unconditionally downloads `main`, which
 # breaks build hermeticity. So, rev-pinning it is.
 # See also https://github.com/bazelbuild/rules_rust/issues/2502.
-codeql-extractor = { git = "https://github.com/github/codeql.git", rev = "bc1283c7152b0bb4d27ff6a004869f493e93d2b3" }
+codeql-extractor = { git = "https://github.com/github/codeql.git", rev = "0dbce3d077f6f31a8d660aea104ee31cacf6bacd" }
 
 [patch.crates-io]
 tree-sitter = {git = "https://github.com/redsun82/tree-sitter.git", rev = "1f5c1112ceaa8fc6aff61d1852690407670d2a96"}