Merge pull request #17808 from github/release-prep/2.19.2

Release preparation for version 2.19.2

cpp/ql/lib/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 2.0.2
+
+### Minor Analysis Improvements
+
+* Added taint flow model for `fopen` and related functions.
+* The `SimpleRangeAnalysis` library (`semmle.code.cpp.rangeanalysis.SimpleRangeAnalysis`) now generates more precise ranges for calls to `fgetc` and `getc`.
+
 ## 2.0.1
 
 No user-facing changes.

cpp/ql/lib/change-notes/2024-10-07-range-analysis-of-getc.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The `SimpleRangeAnalysis` library (`semmle.code.cpp.rangeanalysis.SimpleRangeAnalysis`) now generates more precise ranges for calls to `fgetc` and `getc`.

cpp/ql/lib/change-notes/2024-10-09-fopen-taint.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* Added taint flow model for `fopen` and related functions.

cpp/ql/lib/change-notes/released/2.0.2.md

@@ -0,0 +1,6 @@
+## 2.0.2
+
+### Minor Analysis Improvements
+
+* Added taint flow model for `fopen` and related functions.
+* The `SimpleRangeAnalysis` library (`semmle.code.cpp.rangeanalysis.SimpleRangeAnalysis`) now generates more precise ranges for calls to `fgetc` and `getc`.

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 2.0.1
+lastReleaseVersion: 2.0.2

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 2.0.2-dev
+version: 2.0.2
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/src/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 1.2.5
+
+### Minor Analysis Improvements
+
+* The `cpp/unclear-array-index-validation` ("Unclear validation of array index") query has been improved to reduce false positives and increase true positives.
+* Fixed false positives in the `cpp/uninitialized-local` ("Potentially uninitialized local variable") query if there are extraction errors in the function.
+* The `cpp/incorrect-string-type-conversion` query now produces fewer false positives caused by failure to detect byte arrays.
+* The `cpp/incorrect-string-type-conversion` query now produces fewer false positives caused by failure to recognize dynamic checks prior to possible dangerous widening.
+
 ## 1.2.4
 
 ### Minor Analysis Improvements

cpp/ql/src/change-notes/2024-09-26-wcharcharconversion-false-positives.md

@@ -1,5 +0,0 @@
----
-category: minorAnalysis
----
-* The `cpp/incorrect-string-type-conversion` query now produces fewer false positives caused by failure to detect byte arrays.
-* The `cpp/incorrect-string-type-conversion` query now produces fewer false positives caused by failure to recognize dynamic checks prior to possible dangerous widening.

cpp/ql/src/change-notes/2024-10-02-uninitialized-local.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* Fixed false positives in the `cpp/uninitialized-local` ("Potentially uninitialized local variable") query if there are extraction errors in the function.

cpp/ql/src/change-notes/2024-10-07-unclear-array-index-validation.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The `cpp/unclear-array-index-validation` ("Unclear validation of array index") query has been improved to reduce false positives increase true positives.

cpp/ql/src/change-notes/released/1.2.5.md

@@ -0,0 +1,8 @@
+## 1.2.5
+
+### Minor Analysis Improvements
+
+* The `cpp/unclear-array-index-validation` ("Unclear validation of array index") query has been improved to reduce false positives and increase true positives.
+* Fixed false positives in the `cpp/uninitialized-local` ("Potentially uninitialized local variable") query if there are extraction errors in the function.
+* The `cpp/incorrect-string-type-conversion` query now produces fewer false positives caused by failure to detect byte arrays.
+* The `cpp/incorrect-string-type-conversion` query now produces fewer false positives caused by failure to recognize dynamic checks prior to possible dangerous widening.

cpp/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.2.4
+lastReleaseVersion: 1.2.5

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 1.2.5-dev
+version: 1.2.5
 groups:
   - cpp
   - queries

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetConfig.cs

@@ -1,74 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Text;
-using Semmle.Util;
-
-namespace Semmle.Extraction.CSharp.DependencyFetching
-{
-    public class NugetConfig
-    {
-        internal class NugetFeed : IComparable<NugetFeed>
-        {
-            internal string Value { get; }
-            internal bool DisableTlsCertificateValidation { get; set; }
-
-            internal NugetFeed(string value)
-            {
-                this.Value = value;
-                this.DisableTlsCertificateValidation = true;
-            }
-
-            public override string ToString()
-            {
-                return this.Value;
-            }
-
-            public int CompareTo(NugetFeed? other)
-            {
-                return this.Value.CompareTo(other?.Value);
-            }
-        }
-
-        internal IEnumerable<NugetFeed> Feeds { get; }
-
-        public NugetConfig()
-        {
-            this.Feeds = new List<NugetFeed>();
-        }
-
-        /// <summary>
-        /// Writes this configuration to a file located at <paramref name="nugetConfigPath"/>.
-        /// </summary>
-        /// <param name="nugetConfigPath">The path of the file to which the configuration should be written to.</param>
-        public void Write(string nugetConfigPath)
-        {
-            var config = "";
-
-            var proxyHost = Environment.GetEnvironmentVariable("CODEQL_PROXY_HOST");
-            var proxyPort = Environment.GetEnvironmentVariable("CODEQL_PROXY_PORT");
-            if (!string.IsNullOrWhiteSpace(proxyHost) && !string.IsNullOrWhiteSpace(proxyPort))
-            {
-                var proxyAddress = $"http://{proxyHost}:{proxyPort}";
-                config = $"""<add key="http_proxy" value="{proxyAddress}" />""";
-            }
-
-            var sb = new StringBuilder();
-            this.Feeds.ForEach((feed, index) => sb.AppendLine($"<add key=\"feed{index}\" value=\"{feed.Value}\" disableTLSCertificateValidation=\"{feed.DisableTlsCertificateValidation}\" />"));
-
-            File.WriteAllText(nugetConfigPath,
-                $"""
-                <?xml version="1.0" encoding="utf-8"?>
-                <configuration>
-                    <config>
-                        {config}
-                    </config>
-                    <packageSources>
-                        <clear />
-                        {sb}
-                    </packageSources>
-                </configuration>
-                """);
-        }
-    }
-}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -3,7 +3,6 @@ using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using System.Net;
 using System.Net.Http;
 using System.Security.Cryptography;
 using System.Text;
@@ -94,25 +93,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
         public HashSet<AssemblyLookupLocation> Restore()
         {
-            try
-            {
-                var userConfigPath = Path.Join(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".nuget", "NuGet");
-                Directory.CreateDirectory(userConfigPath);
-
-                var userConfig = new NugetConfig();
-                userConfig.Write(Path.Join(userConfigPath, "NuGet.Config"));
-            }
-            catch (Exception ex)
-            {
-                logger.LogError($"Failed to write user nuget.config: {ex}");
-            }
-
             var assemblyLookupLocations = new HashSet<AssemblyLookupLocation>();
             var checkNugetFeedResponsiveness = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.CheckNugetFeedResponsiveness);
             logger.LogInfo($"Checking NuGet feed responsiveness: {checkNugetFeedResponsiveness}");
             compilationInfoContainer.CompilationInfos.Add(("NuGet feed responsiveness checked", checkNugetFeedResponsiveness ? "1" : "0"));
 
-            HashSet<NugetConfig.NugetFeed>? explicitFeeds = null;
+            HashSet<string>? explicitFeeds = null;
 
             try
             {
@@ -189,12 +175,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return assemblyLookupLocations;
         }
 
-        private List<NugetConfig.NugetFeed> GetReachableFallbackNugetFeeds(HashSet<NugetConfig.NugetFeed>? feedsFromNugetConfigs)
+        private List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNugetConfigs)
         {
-            var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).Select(feed => new NugetConfig.NugetFeed(feed)).ToHashSet();
+            var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet();
             if (fallbackFeeds.Count == 0)
             {
-                fallbackFeeds.Add(new NugetConfig.NugetFeed(PublicNugetOrgFeed));
+                fallbackFeeds.Add(PublicNugetOrgFeed);
                 logger.LogInfo($"No fallback Nuget feeds specified. Adding default feed: {PublicNugetOrgFeed}");
 
                 var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback);
@@ -307,7 +293,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             compilationInfoContainer.CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
         }
 
-        private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds(HashSet<NugetConfig.NugetFeed>? feedsFromNugetConfigs)
+        private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds(HashSet<string>? feedsFromNugetConfigs)
         {
             var reachableFallbackFeeds = GetReachableFallbackNugetFeeds(feedsFromNugetConfigs);
             if (reachableFallbackFeeds.Count > 0)
@@ -319,7 +305,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return null;
         }
 
-        private AssemblyLookupLocation? DownloadMissingPackages(IEnumerable<NugetConfig.NugetFeed>? fallbackNugetFeeds = null)
+        private AssemblyLookupLocation? DownloadMissingPackages(IEnumerable<string>? fallbackNugetFeeds = null)
         {
             var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(PackageDirectory.DirInfo);
             var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
@@ -381,14 +367,23 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return missingPackageDirectory.DirInfo.FullName;
         }
 
-        private string? CreateFallbackNugetConfig(IEnumerable<NugetConfig.NugetFeed> fallbackNugetFeeds, string folderPath)
+        private string? CreateFallbackNugetConfig(IEnumerable<string> fallbackNugetFeeds, string folderPath)
         {
-            NugetConfig config = new NugetConfig();
-            fallbackNugetFeeds.ForEach((feed, index) => config.Feeds.Append(feed));
+            var sb = new StringBuilder();
+            fallbackNugetFeeds.ForEach((feed, index) => sb.AppendLine($"<add key=\"feed{index}\" value=\"{feed}\" />"));
 
             var nugetConfigPath = Path.Combine(folderPath, "nuget.config");
             logger.LogInfo($"Creating fallback nuget.config file {nugetConfigPath}.");
-            config.Write(nugetConfigPath);
+            File.WriteAllText(nugetConfigPath,
+                $"""
+                <?xml version="1.0" encoding="utf-8"?>
+                <configuration>
+                    <packageSources>
+                        <clear />
+                {sb}
+                    </packageSources>
+                </configuration>
+                """);
 
             return nugetConfigPath;
         }
@@ -591,34 +586,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             }
         }
 
-        private bool IsFeedReachable(NugetConfig.NugetFeed feed, int timeoutMilliSeconds, int tryCount, bool allowExceptions = true)
+        private bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount, bool allowExceptions = true)
         {
             logger.LogInfo($"Checking if Nuget feed '{feed}' is reachable...");
-
-            // Configure the handler for this check. If `DisableTlsCertificateValidation` is `true` for this feed,
-            // we disable certificate validation.
-            var handler = new HttpClientHandler();
-
-            var proxyHost = Environment.GetEnvironmentVariable("CODEQL_PROXY_HOST");
-            var proxyPort = Environment.GetEnvironmentVariable("CODEQL_PROXY_PORT");
-            if (!string.IsNullOrWhiteSpace(proxyHost) && !string.IsNullOrWhiteSpace(proxyPort))
-            {
-                var proxyAddress = new Uri($"http://{proxyHost}:{proxyPort}");
-                handler.Proxy = new WebProxy(proxyAddress);
-                handler.Proxy.Credentials = new NetworkCredential(Environment.GetEnvironmentVariable("CODEQL_PROXY_USER"), Environment.GetEnvironmentVariable("CODEQL_PROXY_PASSWORD"));
-
-                logger.LogInfo($"Using proxy at {proxyAddress}...");
-            }
-
-            if (feed.DisableTlsCertificateValidation)
-            {
-                logger.LogInfo($"Disabling TLS certificate validation for '{feed}'...");
-                handler.ClientCertificateOptions = ClientCertificateOption.Manual;
-                handler.ServerCertificateCustomValidationCallback =
-                    (httpRequestMessage, cert, certChain, policyErrors) => { return true; };
-            }
-
-            using HttpClient client = new(handler);
+            using HttpClient client = new();
 
             for (var i = 0; i < tryCount; i++)
             {
@@ -626,7 +597,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 cts.CancelAfter(timeoutMilliSeconds);
                 try
                 {
-                    ExecuteGetRequest(feed.Value, client, cts.Token).GetAwaiter().GetResult();
+                    ExecuteGetRequest(feed, client, cts.Token).GetAwaiter().GetResult();
                     logger.LogInfo($"Querying Nuget feed '{feed}' succeeded.");
                     return true;
                 }
@@ -644,13 +615,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                     // We're only interested in timeouts.
                     var start = allowExceptions ? "Considering" : "Not considering";
                     logger.LogInfo($"Querying Nuget feed '{feed}' failed in a timely manner. {start} the feed for use. The reason for the failure: {exc.Message}");
-                    logger.LogError($"Exception: {exc}");
-
-                    if (exc.InnerException != null)
-                    {
-                        logger.LogError($"Inner exception: {exc.InnerException}");
-                    }
-
                     return allowExceptions;
                 }
             }
@@ -678,7 +642,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return (timeoutMilliSeconds, tryCount);
         }
 
-        private bool CheckFeeds(out HashSet<NugetConfig.NugetFeed> explicitFeeds)
+        private bool CheckFeeds(out HashSet<string> explicitFeeds)
         {
             logger.LogInfo("Checking Nuget feeds...");
             (explicitFeeds, var allFeeds) = GetAllFeeds();
@@ -693,7 +657,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
             var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false);
 
-            var allFeedsReachable = explicitFeeds.All(feed => excludedFeeds.Contains(feed.Value) || IsFeedReachable(feed, initialTimeout, tryCount));
+            var allFeedsReachable = explicitFeeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed, initialTimeout, tryCount));
             if (!allFeedsReachable)
             {
                 logger.LogWarning("Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.");
@@ -747,12 +711,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             }
         }
 
-        private (HashSet<NugetConfig.NugetFeed> explicitFeeds, HashSet<NugetConfig.NugetFeed> allFeeds) GetAllFeeds()
+        private (HashSet<string> explicitFeeds, HashSet<string> allFeeds) GetAllFeeds()
         {
             var nugetConfigs = fileProvider.NugetConfigs;
             var explicitFeeds = nugetConfigs
                 .SelectMany(config => GetFeeds(() => dotnet.GetNugetFeeds(config)))
-                .Select(feed => new NugetConfig.NugetFeed(feed))
                 .ToHashSet();
 
             if (explicitFeeds.Count > 0)
@@ -781,7 +744,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 })
                 .Where(folder => folder != null)
                 .SelectMany(folder => GetFeeds(() => dotnet.GetNugetFeedsFromFolder(folder!)))
-                .Select(feed => new NugetConfig.NugetFeed(feed))
                 .ToHashSet();
 
             logger.LogInfo($"Found {allFeeds.Count} Nuget feeds (with inherited ones) in nuget.config files: {string.Join(", ", allFeeds.OrderBy(f => f))}");

csharp/ql/campaigns/Solorigate/lib/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.7.27
+
+No user-facing changes.
+
 ## 1.7.26
 
 No user-facing changes.

csharp/ql/campaigns/Solorigate/lib/change-notes/released/1.7.27.md

@@ -0,0 +1,3 @@
+## 1.7.27
+
+No user-facing changes.

csharp/ql/campaigns/Solorigate/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.7.26
+lastReleaseVersion: 1.7.27

csharp/ql/campaigns/Solorigate/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-all
-version: 1.7.27-dev
+version: 1.7.27
 groups:
   - csharp
   - solorigate

csharp/ql/campaigns/Solorigate/src/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.7.27
+
+No user-facing changes.
+
 ## 1.7.26
 
 No user-facing changes.

csharp/ql/campaigns/Solorigate/src/change-notes/released/1.7.27.md

@@ -0,0 +1,3 @@
+## 1.7.27
+
+No user-facing changes.

csharp/ql/campaigns/Solorigate/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.7.26
+lastReleaseVersion: 1.7.27

csharp/ql/campaigns/Solorigate/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-queries
-version: 1.7.27-dev
+version: 1.7.27
 groups:
   - csharp
   - solorigate

csharp/ql/lib/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 3.0.1
+
+No user-facing changes.
+
 ## 3.0.0
 
 ### Breaking Changes

csharp/ql/lib/change-notes/released/3.0.1.md

@@ -0,0 +1,3 @@
+## 3.0.1
+
+No user-facing changes.

csharp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 3.0.0
+lastReleaseVersion: 3.0.1

csharp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-all
-version: 3.0.1-dev
+version: 3.0.1
 groups: csharp
 dbscheme: semmlecode.csharp.dbscheme
 extractor: csharp

csharp/ql/src/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.0.10
+
+No user-facing changes.
+
 ## 1.0.9
 
 ### Minor Analysis Improvements

csharp/ql/src/change-notes/released/1.0.10.md

@@ -0,0 +1,3 @@
+## 1.0.10
+
+No user-facing changes.

csharp/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.0.9
+lastReleaseVersion: 1.0.10

csharp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-queries
-version: 1.0.10-dev
+version: 1.0.10
 groups:
   - csharp
   - queries

docs/codeql/index.html

@@ -101,7 +101,7 @@
                             latest version of CodeQL...</div>
                     </div>
                     <div class="Subhead border-0">
-                        <a href="query-help/codeql-cwe-coverage/">
+                        <a href="../codeql-query-help/codeql-cwe-coverage/">
                             <div class="Subhead-heading f4 text-center">CodeQL coverage of CWEs</div>
                         </a>
                         <div class="Subhead-description">Detailed information on the coverage of Common Weakness Enumerations (CWEs) in the latest release...</div>

go/ql/consistency-queries/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.0.10
+
+No user-facing changes.
+
 ## 1.0.9
 
 No user-facing changes.

go/ql/consistency-queries/change-notes/released/1.0.10.md

@@ -0,0 +1,3 @@
+## 1.0.10
+
+No user-facing changes.

go/ql/consistency-queries/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.0.9
+lastReleaseVersion: 1.0.10

go/ql/consistency-queries/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql-go-consistency-queries
-version: 1.0.10-dev
+version: 1.0.10
 groups:
   - go
   - queries

go/ql/lib/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 2.1.1
+
+### Minor Analysis Improvements
+
+* Added member predicates `StructTag.hasOwnFieldWithTag` and `Field.getTag`, which enable CodeQL queries to examine struct field tags.
+* Added member predicate `InterfaceType.hasPrivateMethodWithQualifiedName`, which enables CodeQL queries to distinguish interfaces with matching non-exported method names that are declared in different packages, and are therefore incompatible.
+* Local source models with the `stdin` source kind have been added for the variable `os.Stdin` and the functions `fmt.Scan`, `fmt.Scanf` and `fmt.Scanln`. You can optionally include threat models as appropriate when using the CodeQL CLI and in GitHub code scanning. For more information, see [Analyzing your code with CodeQL queries](https://docs.github.com/code-security/codeql-cli/getting-started-with-the-codeql-cli/analyzing-your-code-with-codeql-queries#including-model-packs-to-add-potential-sources-of-tainted-data>) and [Customizing your advanced setup for code scanning](https://docs.github.com/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning#extending-codeql-coverage-with-threat-models).
+
 ## 2.1.0
 
 ### Deprecated APIs

go/ql/lib/change-notes/2024-09-03-tags-and-interface-ids.md

@@ -1,5 +0,0 @@
----
-category: minorAnalysis
----
-* Added member predicates `StructTag.hasOwnFieldWithTag` and `Field.getTag`, which enable CodeQL queries to examine struct field tags.
-* Added member predicate `InterfaceType.hasPrivateMethodWithQualifiedName`, which enables CodeQL queries to distinguish interfaces with matching non-exported method names that are declared in different packages, and are therefore incompatible.

go/ql/lib/change-notes/released/2.1.1.md

@@ -0,0 +1,7 @@
+## 2.1.1
+
+### Minor Analysis Improvements
+
+* Added member predicates `StructTag.hasOwnFieldWithTag` and `Field.getTag`, which enable CodeQL queries to examine struct field tags.
+* Added member predicate `InterfaceType.hasPrivateMethodWithQualifiedName`, which enables CodeQL queries to distinguish interfaces with matching non-exported method names that are declared in different packages, and are therefore incompatible.
+* Local source models with the `stdin` source kind have been added for the variable `os.Stdin` and the functions `fmt.Scan`, `fmt.Scanf` and `fmt.Scanln`. You can optionally include threat models as appropriate when using the CodeQL CLI and in GitHub code scanning. For more information, see [Analyzing your code with CodeQL queries](https://docs.github.com/code-security/codeql-cli/getting-started-with-the-codeql-cli/analyzing-your-code-with-codeql-queries#including-model-packs-to-add-potential-sources-of-tainted-data>) and [Customizing your advanced setup for code scanning](https://docs.github.com/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning#extending-codeql-coverage-with-threat-models).

go/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 2.1.0
+lastReleaseVersion: 2.1.1

go/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/go-all
-version: 2.1.1-dev
+version: 2.1.1
 groups: go
 dbscheme: go.dbscheme
 extractor: go

go/ql/lib/semmle/go/dataflow/internal/DataFlowNodes.qll

@@ -727,7 +727,10 @@ module Public {
     override string getNodeKind() { result = "external parameter node" }
 
     override Type getType() {
-      result = this.getSummarizedCallable().getType().getParameterType(this.getPos())
+      result =
+        this.getSummarizedCallable()
+            .getType()
+            .getParameterType(pragma[only_bind_into](this.getPos()))
       or
       this.getPos() = -1 and
       result = this.getSummarizedCallable().asFunction().(Method).getReceiverType()

go/ql/lib/semmle/go/frameworks/stdlib/Fmt.qll

@@ -112,6 +112,15 @@ module Fmt {
     Scanner() { this.hasQualifiedName("fmt", ["Scan", "Scanf", "Scanln"]) }
   }
 
+  private class ScannerSource extends SourceNode {
+    ScannerSource() {
+      // All of the arguments which are sources are varargs.
+      this.asExpr() = any(Scanner s).getACall().getAnImplicitVarargsArgument().asExpr()
+    }
+
+    override string getThreatModel() { result = "stdin" }
+  }
+
   /**
    * The `Fscan` function or one of its variants,
    * all of which read from a specified `io.Reader`.

go/ql/lib/semmle/go/frameworks/stdlib/Os.qll

@@ -43,4 +43,12 @@ module Os {
       input = inp and output = outp
     }
   }
+
+  private class Stdin extends SourceNode {
+    Stdin() {
+      exists(Variable osStdin | osStdin.hasQualifiedName("os", "Stdin") | this = osStdin.getARead())
+    }
+
+    override string getThreatModel() { result = "stdin" }
+  }
 }

go/ql/src/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.1.1
+
+No user-facing changes.
+
 ## 1.1.0
 
 ### Query Metadata Changes

go/ql/src/change-notes/released/1.1.1.md

@@ -0,0 +1,3 @@
+## 1.1.1
+
+No user-facing changes.

go/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.1.0
+lastReleaseVersion: 1.1.1

go/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/go-queries
-version: 1.1.1-dev
+version: 1.1.1
 groups:
   - go
   - queries

go/ql/test/library-tests/semmle/go/dataflow/flowsources/local/stdin/go.mod

@@ -0,0 +1,3 @@
+module test
+
+go 1.22.6

go/ql/test/library-tests/semmle/go/dataflow/flowsources/local/stdin/source.expected

@@ -0,0 +1,3 @@
+testFailures
+invalidModelRow
+failures

go/ql/test/library-tests/semmle/go/dataflow/flowsources/local/stdin/source.ext.yml

@@ -0,0 +1,7 @@
+extensions:
+
+  - addsTo:
+      pack: codeql/threat-models
+      extensible: threatModelConfiguration
+    data:
+      - ["stdin", true, 0]

go/ql/test/library-tests/semmle/go/dataflow/flowsources/local/stdin/source.ql

@@ -0,0 +1,19 @@
+import go
+import ModelValidation
+import TestUtilities.InlineExpectationsTest
+
+module SourceTest implements TestSig {
+  string getARelevantTag() { result = "source" }
+
+  predicate hasActualResult(Location location, string element, string tag, string value) {
+    exists(ActiveThreatModelSource s |
+      s.hasLocationInfo(location.getFile().getAbsolutePath(), location.getStartLine(),
+        location.getStartColumn(), location.getEndLine(), location.getEndColumn()) and
+      element = s.toString() and
+      value = "" and
+      tag = "source"
+    )
+  }
+}
+
+import MakeTest<SourceTest>

go/ql/test/library-tests/semmle/go/dataflow/flowsources/local/stdin/test.expected

@@ -0,0 +1,2 @@
+testFailures
+invalidModelRow

go/ql/test/library-tests/semmle/go/dataflow/flowsources/local/stdin/test.ext.yml

@@ -0,0 +1,7 @@
+extensions:
+
+  - addsTo:
+      pack: codeql/threat-models
+      extensible: threatModelConfiguration
+    data:
+      - ["stdin", true, 0]

go/ql/test/library-tests/semmle/go/dataflow/flowsources/local/stdin/test.go

@@ -0,0 +1,48 @@
+package test
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+)
+
+func sink(string) {
+
+}
+
+func readStdinBuffer() {
+	buf := make([]byte, 1024)
+	n, err := os.Stdin.Read(buf) // $source
+	if err != nil {
+		return
+	}
+	sink(string(buf[:n])) // $hasTaintFlow="type conversion"
+}
+
+func readStdinBuffReader() {
+	buf := make([]byte, 1024)
+	r := bufio.NewReader(os.Stdin) // $source
+	n, err := r.Read(buf)
+	if err != nil {
+		return
+	}
+	sink(string(buf[:n])) // $hasTaintFlow="type conversion"
+}
+
+func scan() {
+	var username, email string
+	fmt.Scan(&username, &email) // $source
+	sink(username)              // $hasTaintFlow="username"
+}
+
+func scanf() {
+	var s string
+	fmt.Scanf("%s", &s) // $source
+	sink(s)             // $hasTaintFlow="s"
+}
+
+func scanl() {
+	var s string
+	fmt.Scanln(&s) // $source
+	sink(s)        // $hasTaintFlow="s"
+}

go/ql/test/library-tests/semmle/go/dataflow/flowsources/local/stdin/test.ql

@@ -0,0 +1,15 @@
+import go
+import semmle.go.dataflow.ExternalFlow
+import ModelValidation
+import experimental.frameworks.CleverGo
+import TestUtilities.InlineFlowTest
+
+module Config implements DataFlow::ConfigSig {
+  predicate isSource(DataFlow::Node source) { source instanceof ActiveThreatModelSource }
+
+  predicate isSink(DataFlow::Node sink) {
+    sink.asExpr() = any(CallExpr c | c.getTarget().getName() = "sink").getArgument(0)
+  }
+}
+
+import TaintFlowTest<Config>

java/ql/automodel/src/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.0.10
+
+No user-facing changes.
+
 ## 1.0.9
 
 No user-facing changes.

java/ql/automodel/src/change-notes/released/1.0.10.md

@@ -0,0 +1,3 @@
+## 1.0.10
+
+No user-facing changes.

java/ql/automodel/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.0.9
+lastReleaseVersion: 1.0.10

java/ql/automodel/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-automodel-queries
-version: 1.0.10-dev
+version: 1.0.10
 groups:
     - java
     - automodel

java/ql/lib/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 4.1.1
+
+No user-facing changes.
+
 ## 4.1.0
 
 ### Deprecated APIs

java/ql/lib/change-notes/released/4.1.1.md

@@ -0,0 +1,3 @@
+## 4.1.1
+
+No user-facing changes.

java/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 4.1.0
+lastReleaseVersion: 4.1.1

java/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-all
-version: 4.1.1-dev
+version: 4.1.1
 groups: java
 dbscheme: config/semmlecode.dbscheme
 extractor: java

java/ql/src/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.1.7
+
+No user-facing changes.
+
 ## 1.1.6
 
 ### Minor Analysis Improvements

java/ql/src/change-notes/released/1.1.7.md

@@ -0,0 +1,3 @@
+## 1.1.7
+
+No user-facing changes.

java/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.1.6
+lastReleaseVersion: 1.1.7

java/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-queries
-version: 1.1.7-dev
+version: 1.1.7
 groups:
   - java
   - queries

javascript/ql/lib/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.0.2
+
+No user-facing changes.
+
 ## 2.0.1
 
 No user-facing changes.

javascript/ql/lib/change-notes/released/2.0.2.md

@@ -0,0 +1,3 @@
+## 2.0.2
+
+No user-facing changes.

javascript/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 2.0.1
+lastReleaseVersion: 2.0.2

javascript/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/javascript-all
-version: 2.0.2-dev
+version: 2.0.2
 groups: javascript
 dbscheme: semmlecode.javascript.dbscheme
 extractor: javascript

javascript/ql/src/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.2.2
+
+No user-facing changes.
+
 ## 1.2.1
 
 No user-facing changes.

javascript/ql/src/change-notes/released/1.2.2.md

@@ -0,0 +1,3 @@
+## 1.2.2
+
+No user-facing changes.

javascript/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.2.1
+lastReleaseVersion: 1.2.2

javascript/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/javascript-queries
-version: 1.2.2-dev
+version: 1.2.2
 groups:
   - javascript
   - queries

misc/suite-helpers/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.0.10
+
+No user-facing changes.
+
 ## 1.0.9
 
 No user-facing changes.

misc/suite-helpers/change-notes/released/1.0.10.md

@@ -0,0 +1,3 @@
+## 1.0.10
+
+No user-facing changes.

misc/suite-helpers/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.0.9
+lastReleaseVersion: 1.0.10

misc/suite-helpers/qlpack.yml

@@ -1,4 +1,4 @@
 name: codeql/suite-helpers
-version: 1.0.10-dev
+version: 1.0.10
 groups: shared
 warnOnImplicitThis: true

python/extractor/cli-integration-test/extract-stdlib/test.sh

@@ -13,10 +13,10 @@ rm -rf dbs
 
 mkdir dbs
 
-CODEQL_EXTRACTOR_PYTHON_DONT_EXTRACT_STDLIB=True $CODEQL database create dbs/without-stdlib --language python --source-root repo_dir/
+$CODEQL database create dbs/without-stdlib --language python --source-root repo_dir/
 $CODEQL query run --database dbs/without-stdlib query.ql > query.without-stdlib.actual
 diff query.without-stdlib.expected query.without-stdlib.actual
 
-LGTM_INDEX_EXCLUDE="/usr/lib/**" $CODEQL database create dbs/with-stdlib --language python --source-root repo_dir/
+LGTM_INDEX_EXCLUDE="/usr/lib/**" CODEQL_EXTRACTOR_PYTHON_EXTRACT_STDLIB=True $CODEQL database create dbs/with-stdlib --language python --source-root repo_dir/
 $CODEQL query run --database dbs/with-stdlib query.ql > query.with-stdlib.actual
 diff query.with-stdlib.expected query.with-stdlib.actual

python/extractor/cli-integration-test/python-2-deprecation/query.only-python2.expected

@@ -1,5 +0,0 @@
-|   name   |
-+----------+
-| dircache |
-| stat     |
-| test     |

python/extractor/cli-integration-test/python-2-deprecation/query.python2-using-python3.expected

@@ -1,5 +1,3 @@
-|   name   |
-+----------+
-| dircache |
-| stat     |
-| test     |
+| name |
++------+
+| test |

python/extractor/cli-integration-test/python-2-deprecation/query.without-python2.expected

@@ -1,4 +1,3 @@
 | name |
 +------+
-| stat |
 | test |

python/extractor/semmle/cmdline.py

@@ -102,8 +102,10 @@ def make_parser():
     config_options.add_option("--colorize", dest="colorize", default=False, action="store_true",
                       help = """Colorize the logging output.""")
 
-    config_options.add_option("--dont-extract-stdlib", dest="extract_stdlib", default=True, action="store_false",
-        help="Do not extract the standard library.")
+    config_options.add_option("--dont-extract-stdlib", dest="extract_stdlib", action="store_false",
+        help="This flag is deprecated; not extracting the standard library is now the default.")
+    config_options.add_option("--extract-stdlib", dest="extract_stdlib", default=False, action="store_true",
+        help="Extract the standard library.")
 
     parser.add_option_group(config_options)
 
@@ -226,8 +228,18 @@ def parse(command_line):
 
     if 'CODEQL_EXTRACTOR_PYTHON_DONT_EXTRACT_STDLIB' in os.environ:
         options.extract_stdlib = False
+        print ("WARNING: CODEQL_EXTRACTOR_PYTHON_DONT_EXTRACT_STDLIB is deprecated; the default is now to not extract the standard library.")
+
+    if 'CODEQL_EXTRACTOR_PYTHON_EXTRACT_STDLIB' in os.environ:
+        options.extract_stdlib = True
 
     options.prune = True
+
+    if options.extract_stdlib:
+        print ("WARNING: The analysis will extract the standard library. This behavior is deprecated and will be removed in a future release. We expect it to be gone in CLI version 2.20.0.")
+    else:
+        print ("INFO: The Python extractor has recently stopped extracting the standard library by default. If you encounter problems, please let us know by submitting an issue to https://github.com/github/codeql. It is possible to re-enable extraction of the standard library by setting the environment variable CODEQL_EXTRACTOR_PYTHON_EXTRACT_STDLIB.")
+
     return options, args
 
 def split_and_flatten(options_list, div):

python/extractor/semmle/populator.py

@@ -67,7 +67,7 @@ def main(sys_path = sys.path[:]):
         update_analysis_version(last_version)
 
     found_py2 = False
-    if get_analysis_major_version() == 2:
+    if get_analysis_major_version() == 2 and options.extract_stdlib:
         # Setup `sys_path` to use the Python 2 standard library
         sys_path, found_py2 = get_py2_sys_path(logger, sys_path)
 

python/extractor/semmle/util.py

@@ -10,7 +10,7 @@ from io import BytesIO
 
 #Semantic version of extractor.
 #Update this if any changes are made
-VERSION = "6.1.2"
+VERSION = "7.0.0"
 
 PY_EXTENSIONS = ".py", ".pyw"
 

python/ql/lib/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 2.1.1
+
+### Minor Analysis Improvements
+
+* Modelled that `re.finditer` returns an iterable of `re.Match` objects. This is now understood by the API graph in many cases.
+* Type tracking, and hence the API graph, is now able to correctly trace through comprehensions.
+* More precise modelling of the dataflow through comprehensions. In particular, captured variables are now handled correctly.
+* Dataflow out of yield is added, allowing proper tracing through generators.
+* Added several models of standard library functions and classes, in anticipation of no longer extracting the standard library in a future release.
+
 ## 2.1.0
 
 ### New Features

python/ql/lib/change-notes/2024-09-24-std-lib-models.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* Added several models of standard library functions and classes, in anticipation of no longer extracting the standard library in a future release.

python/ql/lib/change-notes/2024-10-01-comprehension-flow.md

@@ -1,5 +0,0 @@
----
-category: minorAnalysis
----
-* More precise modelling of the dataflow through comprehensions. In particular, captured variables are now handled correctly.
-* Dataflow out of yield is added, allowing proper tracing through generators.

python/ql/lib/change-notes/2024-10-03-typetracking-through-comprehensions.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* Type tracking, and hence the API graph, is now able to correctly trace trough comprehensions.

python/ql/lib/change-notes/2024-10-09-finditer-match.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* Modelled that `re.finditer` returns an iterable of `re.Match` objects. This is now understood by the API graph in many cases.

python/ql/lib/change-notes/released/2.1.1.md

@@ -0,0 +1,9 @@
+## 2.1.1
+
+### Minor Analysis Improvements
+
+* Modelled that `re.finditer` returns an iterable of `re.Match` objects. This is now understood by the API graph in many cases.
+* Type tracking, and hence the API graph, is now able to correctly trace through comprehensions.
+* More precise modelling of the dataflow through comprehensions. In particular, captured variables are now handled correctly.
+* Dataflow out of yield is added, allowing proper tracing through generators.
+* Added several models of standard library functions and classes, in anticipation of no longer extracting the standard library in a future release.

python/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 2.1.0
+lastReleaseVersion: 2.1.1

python/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/python-all
-version: 2.1.1-dev
+version: 2.1.1
 groups: python
 dbscheme: semmlecode.python.dbscheme
 extractor: python

python/ql/lib/semmle/python/frameworks/Stdlib.model.yml

@@ -65,6 +65,10 @@ extensions:
       - ["getopt", "Member[getopt]", "Argument[1,shortopts:,2,longopts:]", "ReturnValue.TupleElement[0].ListElement.TupleElement[0]", "taint"]
       # See https://docs.python.org/3/library/gettext.html#gettext.gettext
       - ["gettext", "Member[gettext]", "Argument[0,message:]", "ReturnValue", "taint"]
+      # See 
+      #  - https://docs.python.org/3/library/glob.html#glob.glob
+      #  - https://docs.python.org/3/library/glob.html#glob.iglob
+      - ["glob", "Member[glob,iglob]", "Argument[0,pathname:]", "ReturnValue", "taint"]
       # See https://docs.python.org/3/library/gzip.html#gzip.GzipFile
       - ["gzip.GzipFile!", "Subclass.Call", "Argument[0,filename:]", "ReturnValue", "taint"]
       # See
@@ -88,6 +92,8 @@ extensions:
       - ["nturl2path", "Member[url2pathname]", "Argument[0,url:]", "ReturnValue", "taint"]
       # See https://docs.python.org/3/library/optparse.html#optparse.OptionParser.parse_args
       - ["optparse.OptionParser", "Member[parse_args]", "Argument[0,args:,1,values:]", "ReturnValue.TupleElement[0,1]", "taint"]
+      # See https://docs.python.org/3/library/os.html#os.walk
+      - ["os", "Member[walk]", "Argument[0,top:]", "ReturnValue", "taint"]
       # See https://github.com/python/cpython/blob/3.10/Lib/pathlib.py#L972-L973
       - ["pathlib.Path", ".Member[__enter__]", "Argument[self]", "ReturnValue", "taint"]
       # See https://docs.python.org/3/library/os.html#os.PathLike.__fspath__

python/ql/src/CHANGELOG.md

@@ -1,8 +1,12 @@
+## 1.3.1
+
+No user-facing changes.
+
 ## 1.3.0
 
 ### New Queries
 
-* The `py/cors-misconfiguration-with-credentials` query, which finds insecure CORS middleware configurations.
+* The experimental `py/cors-misconfiguration-with-credentials` query, which finds insecure CORS middleware configurations.
 
 ## 1.2.2
 

python/ql/src/change-notes/released/1.3.1.md

@@ -0,0 +1,3 @@
+## 1.3.1
+
+No user-facing changes.

python/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.3.0
+lastReleaseVersion: 1.3.1

python/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/python-queries
-version: 1.3.1-dev
+version: 1.3.1
 groups:
   - python
   - queries