Merge branch 'main' into copilot/conversion-of-codeql-queries

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs

@@ -0,0 +1,304 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using Semmle.Util;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    /// <summary>
+    /// Manage the downloading of NuGet packages with nuget.exe.
+    /// Locates packages in a source tree and downloads all of the
+    /// referenced assemblies to a temp folder.
+    /// </summary>
+    internal class NugetExeWrapper : IDisposable
+    {
+        private readonly string? nugetExe;
+        private readonly Semmle.Util.Logging.ILogger logger;
+
+        public int PackageCount => fileProvider.PackagesConfigs.Count;
+
+        private readonly string? backupNugetConfig;
+        private readonly string? nugetConfigPath;
+        private readonly FileProvider fileProvider;
+
+        /// <summary>
+        /// The packages directory.
+        /// This will be in the user-specified or computed Temp location
+        /// so as to not trample the source tree.
+        /// </summary>
+        private readonly DependencyDirectory packageDirectory;
+
+        /// <summary>
+        /// Create the package manager for a specified source tree.
+        /// </summary>
+        public NugetExeWrapper(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, Func<bool> useDefaultFeed)
+        {
+            this.fileProvider = fileProvider;
+            this.packageDirectory = packageDirectory;
+            this.logger = logger;
+
+            if (fileProvider.PackagesConfigs.Count > 0)
+            {
+                logger.LogInfo($"Found packages.config files, trying to use nuget.exe for package restore");
+                nugetExe = ResolveNugetExe();
+                if (HasNoPackageSource() && useDefaultFeed())
+                {
+                    // We only modify or add a top level nuget.config file
+                    nugetConfigPath = Path.Combine(fileProvider.SourceDir.FullName, "nuget.config");
+                    try
+                    {
+                        if (File.Exists(nugetConfigPath))
+                        {
+                            var tempFolderPath = FileUtils.GetTemporaryWorkingDirectory(out _);
+
+                            do
+                            {
+                                backupNugetConfig = Path.Combine(tempFolderPath, Path.GetRandomFileName());
+                            }
+                            while (File.Exists(backupNugetConfig));
+                            File.Copy(nugetConfigPath, backupNugetConfig, true);
+                        }
+                        else
+                        {
+                            File.WriteAllText(nugetConfigPath,
+                                """
+                                <?xml version="1.0" encoding="utf-8"?>
+                                <configuration>
+                                  <packageSources>
+                                  </packageSources>
+                                </configuration>
+                                """);
+                        }
+                        AddDefaultPackageSource(nugetConfigPath);
+                    }
+                    catch (Exception e)
+                    {
+                        logger.LogError($"Failed to add default package source to {nugetConfigPath}: {e}");
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Tries to find the location of `nuget.exe`. It looks for
+        /// - the environment variable specifying a location,
+        /// - files in the repository,
+        /// - tries to resolve nuget from the PATH, or
+        /// - downloads it if it is not found.
+        /// </summary>
+        private string ResolveNugetExe()
+        {
+            var envVarPath = Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetExePath);
+            if (!string.IsNullOrEmpty(envVarPath))
+            {
+                logger.LogInfo($"Using nuget.exe from environment variable: '{envVarPath}'");
+                return envVarPath;
+            }
+
+            try
+            {
+                return DownloadNugetExe(fileProvider.SourceDir.FullName);
+            }
+            catch (Exception exc)
+            {
+                logger.LogInfo($"Download of nuget.exe failed: {exc.Message}");
+            }
+
+            var nugetExesInRepo = fileProvider.NugetExes;
+            if (nugetExesInRepo.Count > 1)
+            {
+                logger.LogInfo($"Found multiple nuget.exe files in the repository: {string.Join(", ", nugetExesInRepo.OrderBy(s => s))}");
+            }
+
+            if (nugetExesInRepo.Count > 0)
+            {
+                var path = nugetExesInRepo.First();
+                logger.LogInfo($"Using nuget.exe from path '{path}'");
+                return path;
+            }
+
+            var executableName = Win32.IsWindows() ? "nuget.exe" : "nuget";
+            var nugetPath = FileUtils.FindProgramOnPath(executableName);
+            if (nugetPath is not null)
+            {
+                nugetPath = Path.Combine(nugetPath, executableName);
+                logger.LogInfo($"Using nuget.exe from PATH: {nugetPath}");
+                return nugetPath;
+            }
+
+            throw new Exception("Could not find or download nuget.exe.");
+        }
+
+        private string DownloadNugetExe(string sourceDir)
+        {
+            var directory = Path.Combine(sourceDir, ".nuget");
+            var nuget = Path.Combine(directory, "nuget.exe");
+
+            // Nuget.exe already exists in the .nuget directory.
+            if (File.Exists(nuget))
+            {
+                logger.LogInfo($"Found nuget.exe at {nuget}");
+                return nuget;
+            }
+
+            Directory.CreateDirectory(directory);
+            logger.LogInfo("Attempting to download nuget.exe");
+            FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget, logger);
+            logger.LogInfo($"Downloaded nuget.exe to {nuget}");
+            return nuget;
+        }
+
+        private bool RunWithMono => !Win32.IsWindows() && !string.IsNullOrEmpty(Path.GetExtension(nugetExe));
+
+        /// <summary>
+        /// Restore all packages in the specified packages.config file.
+        /// </summary>
+        /// <param name="packagesConfig">The packages.config file.</param>
+        private bool TryRestoreNugetPackage(string packagesConfig)
+        {
+            logger.LogInfo($"Restoring file \"{packagesConfig}\"...");
+
+            /* Use nuget.exe to install a package.
+             * Note that there is a clutch of NuGet assemblies which could be used to
+             * invoke this directly, which would arguably be nicer. However they are
+             * really unwieldy and this solution works for now.
+             */
+
+            string exe, args;
+            if (RunWithMono)
+            {
+                exe = "mono";
+                args = $"\"{nugetExe}\" install -OutputDirectory \"{packageDirectory}\" \"{packagesConfig}\"";
+            }
+            else
+            {
+                exe = nugetExe!;
+                args = $"install -OutputDirectory \"{packageDirectory}\" \"{packagesConfig}\"";
+            }
+
+            var pi = new ProcessStartInfo(exe, args)
+            {
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                UseShellExecute = false
+            };
+
+            var threadId = Environment.CurrentManagedThreadId;
+            void onOut(string s) => logger.LogDebug(s, threadId);
+            void onError(string s) => logger.LogError(s, threadId);
+            var exitCode = pi.ReadOutput(out _, onOut, onError);
+            if (exitCode != 0)
+            {
+                logger.LogError($"Command {pi.FileName} {pi.Arguments} failed with exit code {exitCode}");
+                return false;
+            }
+            else
+            {
+                logger.LogInfo($"Restored file \"{packagesConfig}\"");
+                return true;
+            }
+        }
+
+        /// <summary>
+        /// Download the packages to the temp folder.
+        /// </summary>
+        public int InstallPackages()
+        {
+            return fileProvider.PackagesConfigs.Count(TryRestoreNugetPackage);
+        }
+
+        private bool HasNoPackageSource()
+        {
+            if (Win32.IsWindows())
+            {
+                return false;
+            }
+
+            try
+            {
+                logger.LogInfo("Checking if default package source is available...");
+                RunMonoNugetCommand("sources list -ForceEnglishOutput", out var stdout);
+                if (stdout.All(line => line != "No sources found."))
+                {
+                    return false;
+                }
+
+                return true;
+            }
+            catch (Exception e)
+            {
+                logger.LogWarning($"Failed to check if default package source is added: {e}");
+                return false;
+            }
+        }
+
+        private void RunMonoNugetCommand(string command, out IList<string> stdout)
+        {
+            string exe, args;
+            if (RunWithMono)
+            {
+                exe = "mono";
+                args = $"\"{nugetExe}\" {command}";
+            }
+            else
+            {
+                exe = nugetExe!;
+                args = command;
+            }
+
+            var pi = new ProcessStartInfo(exe, args)
+            {
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                UseShellExecute = false
+            };
+
+            var threadId = Environment.CurrentManagedThreadId;
+            void onOut(string s) => logger.LogDebug(s, threadId);
+            void onError(string s) => logger.LogError(s, threadId);
+            pi.ReadOutput(out stdout, onOut, onError);
+        }
+
+        private void AddDefaultPackageSource(string nugetConfig)
+        {
+            logger.LogInfo("Adding default package source...");
+            RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source {NugetPackageRestorer.PublicNugetOrgFeed} -ConfigFile \"{nugetConfig}\"", out _);
+        }
+
+        public void Dispose()
+        {
+            if (nugetConfigPath is null)
+            {
+                return;
+            }
+
+            try
+            {
+                if (backupNugetConfig is null)
+                {
+                    logger.LogInfo("Removing nuget.config file");
+                    File.Delete(nugetConfigPath);
+                    return;
+                }
+
+                logger.LogInfo("Reverting nuget.config file content");
+                // The content of the original nuget.config file is reverted without changing the file's attributes or casing:
+                using (var backup = File.OpenRead(backupNugetConfig))
+                using (var current = File.OpenWrite(nugetConfigPath))
+                {
+                    current.SetLength(0);   // Truncate file
+                    backup.CopyTo(current); // Restore original content
+                }
+
+                logger.LogInfo("Deleting backup nuget.config file");
+                File.Delete(backupNugetConfig);
+            }
+            catch (Exception exc)
+            {
+                logger.LogError($"Failed to restore original nuget.config file: {exc}");
+            }
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -161,13 +161,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                     reachableFeeds.UnionWith(reachableInheritedFeeds);
                 }
 
-                using (var packagesConfigRestore = PackagesConfigRestoreFactory.Create(fileProvider, legacyPackageDirectory, logger, IsDefaultFeedReachable))
+                using (var nuget = new NugetExeWrapper(fileProvider, legacyPackageDirectory, logger, IsDefaultFeedReachable))
                 {
-                    var count = packagesConfigRestore.InstallPackages();
+                    var count = nuget.InstallPackages();
 
-                    if (packagesConfigRestore.PackageCount > 0)
+                    if (nuget.PackageCount > 0)
                     {
-                        compilationInfoContainer.CompilationInfos.Add(("packages.config files", packagesConfigRestore.PackageCount.ToString()));
+                        compilationInfoContainer.CompilationInfos.Add(("packages.config files", nuget.PackageCount.ToString()));
                         compilationInfoContainer.CompilationInfos.Add(("Successfully restored packages.config files", count.ToString()));
                     }
                 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs

@@ -1,368 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.IO;
-using System.Linq;
-using Semmle.Util;
-
-namespace Semmle.Extraction.CSharp.DependencyFetching
-{
-    internal interface IPackagesConfigRestore : IDisposable
-    {
-        /// <summary>
-        /// The number of packages.config files found in the source tree.
-        /// </summary>
-        int PackageCount { get; }
-
-        /// <summary>
-        /// Download the packages to the temp folder.
-        /// </summary>
-        int InstallPackages();
-    }
-
-    /// <summary>
-    /// Factory for creating a package manager to restore NuGet packages referenced in packages.config files.
-    /// If the environment doesn't support using nuget.exe to restore packages from packages.config files, a no-op implementation is returned.
-    /// It is worth noting that for macOS and Linux, nuget.exe is used with mono. However, mono is being deprecated and the last GitHub images
-    /// to contain mono are:
-    /// - Ubuntu 22.04
-    /// - macOS 14
-    ///
-    /// If the packages from the packages.config files are not restored with the packages.config restore functionality below, there is a subsequent
-    /// step that still may succeed in restoring the packages without the help of nuget.exe (by attempting to restore using dotnet).
-    /// </summary>
-    internal class PackagesConfigRestoreFactory
-    {
-        public static IPackagesConfigRestore Create(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, Func<bool> useDefaultFeed)
-        {
-            if (SystemBuildActions.Instance.IsWindows() || SystemBuildActions.Instance.IsMonoInstalled())
-            {
-                return new NugetExeWrapper(fileProvider, packageDirectory, logger, useDefaultFeed);
-            }
-
-            return new NoOpPackagesConfig(fileProvider, logger);
-        }
-
-        /// <summary>
-        /// Manage the downloading of NuGet packages with nuget.exe.
-        /// Locates packages in a source tree and downloads all of the
-        /// referenced assemblies to a temp folder.
-        /// </summary>
-        private class NugetExeWrapper : IPackagesConfigRestore
-        {
-            private readonly string? nugetExe;
-            private readonly Semmle.Util.Logging.ILogger logger;
-
-            public int PackageCount => fileProvider.PackagesConfigs.Count;
-
-            private readonly string? backupNugetConfig;
-            private readonly string? nugetConfigPath;
-            private readonly FileProvider fileProvider;
-
-            /// <summary>
-            /// The packages directory.
-            /// This will be in the user-specified or computed Temp location
-            /// so as to not trample the source tree.
-            /// </summary>
-            private readonly DependencyDirectory packageDirectory;
-
-            private bool IsWindows => SystemBuildActions.Instance.IsWindows();
-
-            /// <summary>
-            /// Create the package manager for a specified source tree.
-            /// </summary>
-            public NugetExeWrapper(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, Func<bool> useDefaultFeed)
-            {
-                this.fileProvider = fileProvider;
-                this.packageDirectory = packageDirectory;
-                this.logger = logger;
-
-                if (fileProvider.PackagesConfigs.Count > 0)
-                {
-                    logger.LogInfo($"Found packages.config files, trying to use nuget.exe for package restore");
-                    nugetExe = ResolveNugetExe();
-                    if (!HasPackageSource() && useDefaultFeed())
-                    {
-                        // We only modify or add a top level nuget.config file
-                        nugetConfigPath = Path.Join(fileProvider.SourceDir.FullName, "nuget.config");
-                        try
-                        {
-                            if (File.Exists(nugetConfigPath))
-                            {
-                                var tempFolderPath = FileUtils.GetTemporaryWorkingDirectory(out _);
-
-                                do
-                                {
-                                    backupNugetConfig = Path.Join(tempFolderPath, Path.GetRandomFileName());
-                                }
-                                while (File.Exists(backupNugetConfig));
-                                File.Copy(nugetConfigPath, backupNugetConfig, true);
-                            }
-                            else
-                            {
-                                File.WriteAllText(nugetConfigPath,
-                                    """
-                                <?xml version="1.0" encoding="utf-8"?>
-                                <configuration>
-                                  <packageSources>
-                                  </packageSources>
-                                </configuration>
-                                """);
-                            }
-                            AddDefaultPackageSource(nugetConfigPath);
-                        }
-                        catch (Exception e)
-                        {
-                            logger.LogError($"Failed to add default package source to {nugetConfigPath}: {e}");
-                        }
-                    }
-                }
-            }
-
-            /// <summary>
-            /// Tries to find the location of `nuget.exe`. It looks for
-            /// - the environment variable specifying a location,
-            /// - files in the repository,
-            /// - tries to resolve nuget from the PATH, or
-            /// - downloads it if it is not found.
-            /// </summary>
-            private string ResolveNugetExe()
-            {
-                var envVarPath = Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetExePath);
-                if (!string.IsNullOrEmpty(envVarPath))
-                {
-                    logger.LogInfo($"Using nuget.exe from environment variable: '{envVarPath}'");
-                    return envVarPath;
-                }
-
-                try
-                {
-                    return DownloadNugetExe(fileProvider.SourceDir.FullName);
-                }
-                catch (Exception exc)
-                {
-                    logger.LogInfo($"Download of nuget.exe failed: {exc.Message}");
-                }
-
-                var nugetExesInRepo = fileProvider.NugetExes;
-                if (nugetExesInRepo.Count > 1)
-                {
-                    logger.LogInfo($"Found multiple nuget.exe files in the repository: {string.Join(", ", nugetExesInRepo.OrderBy(s => s))}");
-                }
-
-                if (nugetExesInRepo.Count > 0)
-                {
-                    var path = nugetExesInRepo.First();
-                    logger.LogInfo($"Using nuget.exe from path '{path}'");
-                    return path;
-                }
-
-                var executableName = IsWindows ? "nuget.exe" : "nuget";
-                var nugetPath = FileUtils.FindProgramOnPath(executableName);
-                if (nugetPath is not null)
-                {
-                    nugetPath = Path.Join(nugetPath, executableName);
-                    logger.LogInfo($"Using nuget.exe from PATH: {nugetPath}");
-                    return nugetPath;
-                }
-
-                throw new Exception("Could not find or download nuget.exe.");
-            }
-
-            private string DownloadNugetExe(string sourceDir)
-            {
-                var directory = Path.Join(sourceDir, ".nuget");
-                var nuget = Path.Join(directory, "nuget.exe");
-
-                // Nuget.exe already exists in the .nuget directory.
-                if (File.Exists(nuget))
-                {
-                    logger.LogInfo($"Found nuget.exe at {nuget}");
-                    return nuget;
-                }
-
-                Directory.CreateDirectory(directory);
-                logger.LogInfo("Attempting to download nuget.exe");
-                FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget, logger);
-                logger.LogInfo($"Downloaded nuget.exe to {nuget}");
-                return nuget;
-            }
-
-            private bool RunWithMono => !IsWindows && !string.IsNullOrEmpty(Path.GetExtension(nugetExe));
-
-            /// <summary>
-            /// Restore all packages in the specified packages.config file.
-            /// </summary>
-            /// <param name="packagesConfig">The packages.config file.</param>
-            private bool TryRestoreNugetPackage(string packagesConfig)
-            {
-                logger.LogInfo($"Restoring file \"{packagesConfig}\"...");
-
-                /* Use nuget.exe to install a package.
-                 * Note that there is a clutch of NuGet assemblies which could be used to
-                 * invoke this directly, which would arguably be nicer. However they are
-                 * really unwieldy and this solution works for now.
-                 */
-
-                string exe, args;
-                if (RunWithMono)
-                {
-                    exe = "mono";
-                    args = $"\"{nugetExe}\" install -OutputDirectory \"{packageDirectory}\" \"{packagesConfig}\"";
-                }
-                else
-                {
-                    exe = nugetExe!;
-                    args = $"install -OutputDirectory \"{packageDirectory}\" \"{packagesConfig}\"";
-                }
-
-                var pi = new ProcessStartInfo(exe, args)
-                {
-                    RedirectStandardOutput = true,
-                    RedirectStandardError = true,
-                    UseShellExecute = false
-                };
-
-                var threadId = Environment.CurrentManagedThreadId;
-                void onOut(string s) => logger.LogDebug(s, threadId);
-                void onError(string s) => logger.LogError(s, threadId);
-                var exitCode = pi.ReadOutput(out _, onOut, onError);
-                if (exitCode != 0)
-                {
-                    logger.LogError($"Command {pi.FileName} {pi.Arguments} failed with exit code {exitCode}");
-                    return false;
-                }
-                else
-                {
-                    logger.LogInfo($"Restored file \"{packagesConfig}\"");
-                    return true;
-                }
-            }
-
-            /// <summary>
-            /// Download the packages to the temp folder.
-            /// </summary>
-            public int InstallPackages()
-            {
-                return fileProvider.PackagesConfigs.Count(TryRestoreNugetPackage);
-            }
-
-            private bool HasPackageSource()
-            {
-                if (IsWindows)
-                {
-                    return true;
-                }
-
-                try
-                {
-                    logger.LogInfo("Checking if default package source is available...");
-                    RunMonoNugetCommand("sources list -ForceEnglishOutput", out var stdout);
-                    if (stdout.All(line => line != "No sources found."))
-                    {
-                        return true;
-                    }
-
-                    return false;
-                }
-                catch (Exception e)
-                {
-                    logger.LogWarning($"Failed to check if default package source is added: {e}");
-                    return true;
-                }
-            }
-
-            private void RunMonoNugetCommand(string command, out IList<string> stdout)
-            {
-                string exe, args;
-                if (RunWithMono)
-                {
-                    exe = "mono";
-                    args = $"\"{nugetExe}\" {command}";
-                }
-                else
-                {
-                    exe = nugetExe!;
-                    args = command;
-                }
-
-                var pi = new ProcessStartInfo(exe, args)
-                {
-                    RedirectStandardOutput = true,
-                    RedirectStandardError = true,
-                    UseShellExecute = false
-                };
-
-                var threadId = Environment.CurrentManagedThreadId;
-                void onOut(string s) => logger.LogDebug(s, threadId);
-                void onError(string s) => logger.LogError(s, threadId);
-                pi.ReadOutput(out stdout, onOut, onError);
-            }
-
-            private void AddDefaultPackageSource(string nugetConfig)
-            {
-                logger.LogInfo("Adding default package source...");
-                RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source {NugetPackageRestorer.PublicNugetOrgFeed} -ConfigFile \"{nugetConfig}\"", out _);
-            }
-
-            public void Dispose()
-            {
-                if (nugetConfigPath is null)
-                {
-                    return;
-                }
-
-                try
-                {
-                    if (backupNugetConfig is null)
-                    {
-                        logger.LogInfo("Removing nuget.config file");
-                        File.Delete(nugetConfigPath);
-                        return;
-                    }
-
-                    logger.LogInfo("Reverting nuget.config file content");
-                    // The content of the original nuget.config file is reverted without changing the file's attributes or casing:
-                    using (var backup = File.OpenRead(backupNugetConfig))
-                    using (var current = File.OpenWrite(nugetConfigPath))
-                    {
-                        current.SetLength(0);   // Truncate file
-                        backup.CopyTo(current); // Restore original content
-                    }
-
-                    logger.LogInfo("Deleting backup nuget.config file");
-                    File.Delete(backupNugetConfig);
-                }
-                catch (Exception exc)
-                {
-                    logger.LogError($"Failed to restore original nuget.config file: {exc}");
-                }
-            }
-        }
-
-        private class NoOpPackagesConfig : IPackagesConfigRestore
-        {
-            private readonly Semmle.Util.Logging.ILogger logger;
-            private readonly FileProvider fileProvider;
-
-            public NoOpPackagesConfig(FileProvider fileProvider, Semmle.Util.Logging.ILogger logger)
-            {
-                this.fileProvider = fileProvider;
-                this.logger = logger;
-            }
-
-            public int PackageCount => fileProvider.PackagesConfigs.Count;
-
-            public int InstallPackages()
-            {
-                if (PackageCount > 0)
-                {
-                    logger.LogInfo("Found packages.config files, but nuget.exe cannot be used to restore packages on this platform. Skipping restore of packages.config files.");
-                }
-                return 0;
-            }
-
-            public void Dispose() { }
-        }
-    }
-}

csharp/ql/lib/semmle/code/csharp/controlflow/internal/ControlFlowGraph.qll

@@ -145,8 +145,6 @@ module Ast implements AstSig<Location> {
   final private class ParameterFinal = CS::Parameter;
 
   class Parameter extends ParameterFinal {
-    AstNode getPattern() { result = this }
-
     Expr getDefaultValue() {
       // Avoid combinatorial explosions for callables with multiple bodies
       result = unique( | | super.getDefaultValue())

go/documentation/library-coverage/coverage.csv

@@ -123,7 +123,7 @@ k8s.io/api/core,,,10,,,,,,,,,,,,,,,,,,,,,,,10,
 k8s.io/apimachinery/pkg/runtime,,,47,,,,,,,,,,,,,,,,,,,,,,,47,
 k8s.io/klog,90,,,,,,90,,,,,,,,,,,,,,,,,,,,
 launchpad.net/xmlpath,2,,,,,,,,,,,,,,,,,,2,,,,,,,,
-log,40,,3,,,,40,,,,,,,,,,,,,,,,,,,3,
+log,20,,3,,,,20,,,,,,,,,,,,,,,,,,,3,
 math/big,,,1,,,,,,,,,,,,,,,,,,,,,,,1,
 mime,,,14,,,,,,,,,,,,,,,,,,,,,,,14,
 net,2,16,100,,,,,,1,,,,,,,,1,,,,,,,16,,100,

go/documentation/library-coverage/coverage.rst

@@ -32,7 +32,7 @@ Go framework & library support
    `Revel <http://revel.github.io/>`_,"``github.com/revel/revel*``, ``github.com/robfig/revel*``",46,20,4
    `SendGrid <https://github.com/sendgrid/sendgrid-go>`_,``github.com/sendgrid/sendgrid-go*``,,1,
    `Squirrel <https://github.com/Masterminds/squirrel>`_,"``github.com/Masterminds/squirrel*``, ``github.com/lann/squirrel*``, ``gopkg.in/Masterminds/squirrel``",81,,96
-   `Standard library <https://pkg.go.dev/std>`_,"````, ``archive/*``, ``bufio``, ``bytes``, ``cmp``, ``compress/*``, ``container/*``, ``context``, ``crypto``, ``crypto/*``, ``database/*``, ``debug/*``, ``embed``, ``encoding``, ``encoding/*``, ``errors``, ``expvar``, ``flag``, ``fmt``, ``go/*``, ``hash``, ``hash/*``, ``html``, ``html/*``, ``image``, ``image/*``, ``index/*``, ``io``, ``io/*``, ``log``, ``log/*``, ``maps``, ``math``, ``math/*``, ``mime``, ``mime/*``, ``net``, ``net/*``, ``os``, ``os/*``, ``path``, ``path/*``, ``plugin``, ``reflect``, ``reflect/*``, ``regexp``, ``regexp/*``, ``slices``, ``sort``, ``strconv``, ``strings``, ``sync``, ``sync/*``, ``syscall``, ``syscall/*``, ``testing``, ``testing/*``, ``text/*``, ``time``, ``time/*``, ``unicode``, ``unicode/*``, ``unsafe``, ``weak``",52,612,124
+   `Standard library <https://pkg.go.dev/std>`_,"````, ``archive/*``, ``bufio``, ``bytes``, ``cmp``, ``compress/*``, ``container/*``, ``context``, ``crypto``, ``crypto/*``, ``database/*``, ``debug/*``, ``embed``, ``encoding``, ``encoding/*``, ``errors``, ``expvar``, ``flag``, ``fmt``, ``go/*``, ``hash``, ``hash/*``, ``html``, ``html/*``, ``image``, ``image/*``, ``index/*``, ``io``, ``io/*``, ``log``, ``log/*``, ``maps``, ``math``, ``math/*``, ``mime``, ``mime/*``, ``net``, ``net/*``, ``os``, ``os/*``, ``path``, ``path/*``, ``plugin``, ``reflect``, ``reflect/*``, ``regexp``, ``regexp/*``, ``slices``, ``sort``, ``strconv``, ``strings``, ``sync``, ``sync/*``, ``syscall``, ``syscall/*``, ``testing``, ``testing/*``, ``text/*``, ``time``, ``time/*``, ``unicode``, ``unicode/*``, ``unsafe``, ``weak``",52,612,104
    `XORM <https://xorm.io>`_,"``github.com/go-xorm/xorm*``, ``xorm.io/xorm*``",,,68
    `XPath <https://github.com/antchfx/xpath>`_,``github.com/antchfx/xpath*``,,,4
    `appleboy/gin-jwt <https://github.com/appleboy/gin-jwt>`_,``github.com/appleboy/gin-jwt*``,,,1
@@ -74,5 +74,5 @@ Go framework & library support
    `xpathparser <https://github.com/santhosh-tekuri/xpathparser>`_,``github.com/santhosh-tekuri/xpathparser*``,,,2
    `yaml <https://gopkg.in/yaml.v3>`_,``gopkg.in/yaml*``,,9,
    `zap <https://go.uber.org/zap>`_,``go.uber.org/zap*``,,11,33
-   Totals,,688,1072,1577
+   Totals,,688,1072,1557
 

java/ql/lib/semmle/code/java/ControlFlowGraph.qll

@@ -61,8 +61,6 @@ private module Ast implements AstSig<Location> {
   class Parameter extends AstNode {
     Parameter() { none() }
 
-    AstNode getPattern() { none() }
-
     Expr getDefaultValue() { none() }
   }
 

python/ql/test/2/query-tests/Classes/inconsistent-mro/InconsistentMRO.qlref

@@ -1 +1,2 @@
-Classes/InconsistentMRO.ql
+query: Classes/InconsistentMRO.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Classes/inconsistent-mro/inconsistent_mro.py

@@ -6,7 +6,7 @@ class X(object):
 class Y(X):
     pass
 
-class Z(X, Y):
+class Z(X, Y): # $ Alert
     pass
     
 class O:

python/ql/test/2/query-tests/Classes/new-style/PropertyInOldStyleClass.qlref

@@ -1 +1,2 @@
-Classes/PropertyInOldStyleClass.ql
+query: Classes/PropertyInOldStyleClass.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Classes/new-style/SlotsInOldStyleClass.qlref

@@ -1 +1,2 @@
-Classes/SlotsInOldStyleClass.ql
+query: Classes/SlotsInOldStyleClass.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Classes/new-style/SuperInOldStyleClass.qlref

@@ -1 +1,2 @@
-Classes/SuperInOldStyleClass.ql
+query: Classes/SuperInOldStyleClass.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Classes/new-style/newstyle_test.py

@@ -1,7 +1,7 @@
 
 #Only works for Python2
 
-class OldStyle1:
+class OldStyle1: # $ Alert[py/slots-in-old-style-class]
 
     __slots__ = [ 'a', 'b' ]
 
@@ -12,7 +12,7 @@ class OldStyle1:
 class OldStyle2:
 
     def __init__(self, x):
-        super().__init__(x)
+        super().__init__(x) # $ Alert[py/super-in-old-style]
 
 class NewStyle1(object):
 

python/ql/test/2/query-tests/Classes/new-style/property_old_style.py

@@ -5,6 +5,6 @@ class OldStyle:
     def __init__(self, x):
         self._x = x
 
-    @property
+    @property # $ Alert[py/property-in-old-style-class]
     def piosc(self):
         return self._x

python/ql/test/2/query-tests/Classes/undefined-attribute/MaybeUndefinedClassAttribute.qlref

@@ -1 +1,2 @@
-Classes/MaybeUndefinedClassAttribute.ql
+query: Classes/MaybeUndefinedClassAttribute.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Classes/undefined-attribute/UndefinedClassAttribute.qlref

@@ -1 +1,2 @@
-Classes/UndefinedClassAttribute.ql
+query: Classes/UndefinedClassAttribute.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Exceptions/general/CatchingBaseException.qlref

@@ -1 +1,2 @@
-Exceptions/CatchingBaseException.ql
+query: Exceptions/CatchingBaseException.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Exceptions/general/EmptyExcept.qlref

@@ -1 +1,2 @@
-Exceptions/EmptyExcept.ql
+query: Exceptions/EmptyExcept.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Exceptions/general/IllegalExceptionHandlerType.qlref

@@ -1 +1,2 @@
-Exceptions/IllegalExceptionHandlerType.ql
+query: Exceptions/IllegalExceptionHandlerType.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Exceptions/general/IllegalRaise.qlref

@@ -1 +1,2 @@
-Exceptions/IllegalRaise.ql
+query: Exceptions/IllegalRaise.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Exceptions/general/IncorrectExceptOrder.qlref

@@ -1 +1,2 @@
-Exceptions/IncorrectExceptOrder.ql
+query: Exceptions/IncorrectExceptOrder.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Exceptions/general/exceptions_test.py

@@ -14,4 +14,4 @@ def raise_tuple(cond):
         raise (Exception, "bananas", 17)
     else:
         #This is an error
-        raise (17, "bananas", Exception)
+        raise (17, "bananas", Exception) # $ Alert[py/illegal-raise]

python/ql/test/2/query-tests/Exceptions/generators/UnguardedNextInGenerator.qlref

@@ -1 +1,2 @@
-Exceptions/UnguardedNextInGenerator.ql
+query: Exceptions/UnguardedNextInGenerator.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Exceptions/generators/test.py

@@ -2,12 +2,12 @@
 
 def bad1(it):
     while True:
-        yield next(it)
+        yield next(it) # $ Alert
 
 def bad2(seq):
     it = iter(seq)
     #Not OK as seq  may be empty
-    raise KeyError(next(it))
+    raise KeyError(next(it)) # $ Alert
     yield 0
 
 def ok1(seq):

python/ql/test/2/query-tests/Exceptions/raising/RaisingTuple.qlref

@@ -1 +1,2 @@
-Exceptions/RaisingTuple.ql
+query: Exceptions/RaisingTuple.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Exceptions/raising/test.py

@@ -5,11 +5,11 @@ def ok():
 
 def bad1():
     ex = Exception, "message"
-    raise ex
+    raise ex # $ Alert
 
 def bad2():
-    raise (Exception, "message")
+    raise (Exception, "message") # $ Alert
 
 def bad3():
     ex = Exception, 
-    raise ex, "message"
+    raise ex, "message" # $ Alert

python/ql/test/2/query-tests/Expressions/TruncatedDivision.qlref

@@ -1 +1,2 @@
-Expressions/TruncatedDivision.ql
+query: Expressions/TruncatedDivision.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Expressions/TruncatedDivision_test.py

@@ -62,14 +62,14 @@ print(average([1.0, 2.0]))
 # This case is bad, and is a minimal obvious case that should be bad. It
 # SHOULD be found by the query.
 
-print(3 / 2)
+print(3 / 2) # $ Alert[py/truncated-division]
 
 
 
 # This case is bad. It uses indirect returns of integers through function calls
 # to produce the problem. I
 
-print(return_three() / return_two())
+print(return_three() / return_two()) # $ Alert[py/truncated-division]
 
 
 

python/ql/test/2/query-tests/Expressions/UseofApply.py

@@ -16,7 +16,7 @@ def useofapply():
 
   # This use of `apply` is a reference to the builtin function and so SHOULD be
   # caught by the query.
-  apply(foo, [1])
+  apply(foo, [1]) # $ Alert[py/use-of-apply]
 
 
 

python/ql/test/2/query-tests/Expressions/UseofApply.qlref

@@ -1 +1,2 @@
-Expressions/UseofApply.ql
+query: Expressions/UseofApply.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Expressions/UseofInput.qlref

@@ -1 +1,2 @@
-Expressions/UseofInput.ql
+query: Expressions/UseofInput.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Expressions/expressions_test.py

@@ -1,9 +1,9 @@
 def use_of_apply(func, args):
-    apply(func, args)
+    apply(func, args) # $ Alert[py/use-of-apply]
 
 
 def use_of_input():
-    return input() # NOT OK
+    return input() # $ Alert[py/use-of-input] # NOT OK
 
 
 def not_use_of_input():

python/ql/test/2/query-tests/Functions/DeprecatedSliceMethod.qlref

@@ -1 +1,2 @@
-Functions/DeprecatedSliceMethod.ql
+query: Functions/DeprecatedSliceMethod.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Imports/encoding_error/EncodingError.qlref

@@ -1 +1,2 @@
-Imports/EncodingError.ql
+query: Imports/EncodingError.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Imports/syntax_error/EncodingError.qlref

@@ -1 +1,2 @@
-Imports/EncodingError.ql
+query: Imports/EncodingError.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Imports/syntax_error/SyntaxError.qlref

@@ -1 +1,2 @@
-Imports/SyntaxError.ql
+query: Imports/SyntaxError.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Imports/syntax_error/bad_encoding.py

@@ -8,5 +8,5 @@
 # encoding:shift-jis
 
 def f():
-    print "Python <20>̊J<CC8A><4A><EFBFBD>́A1990 <20>N<EFBFBD><4E><EFBFBD>납<EFBFBD><EB82A9><EFBFBD>J<EFBFBD>n<EFBFBD><6E><EFBFBD><EFBFBD><EFBFBD>Ă<EFBFBD><C482>܂<EFBFBD>"
+    print "Python <20>̊J<CC8A><4A><EFBFBD>́A1990 <20>N<EFBFBD><4E><EFBFBD>납<EFBFBD><EB82A9><EFBFBD>J<EFBFBD>n<EFBFBD><6E><EFBFBD><EFBFBD><EFBFBD>Ă<EFBFBD><C482>܂<EFBFBD>" # $ Alert[py/encoding-error]
 """

python/ql/test/2/query-tests/Imports/syntax_error/nonsense.py

@@ -1,4 +1,4 @@
-`Twas brillig, and the slithy toves
+`Twas brillig, and the slithy toves # $ Alert[py/syntax-error]
   Did gyre and gimble in the wabe:
 All mimsy were the borogoves,
   And the mome raths outgrabe.

python/ql/test/2/query-tests/Lexical/OldOctalLiteral.qlref

@@ -1 +1,2 @@
-Lexical/OldOctalLiteral.ql
+query: Lexical/OldOctalLiteral.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Lexical/lexical_test.py

@@ -1,6 +1,6 @@
 
 #Bad Octal literal
-017
+017 # $ Alert
 #Good Octal literal
 0o17
 #Special case file permissions

python/ql/test/2/query-tests/Statements/ExecUsed.qlref

@@ -1 +1,2 @@
-Statements/ExecUsed.ql
+query: Statements/ExecUsed.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Statements/IterableStringOrSequence.qlref

@@ -1 +1,2 @@
-Statements/IterableStringOrSequence.ql
+query: Statements/IterableStringOrSequence.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Statements/TopLevelPrint.qlref

@@ -1 +1,2 @@
-Statements/TopLevelPrint.ql
+query: Statements/TopLevelPrint.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Statements/module.py

@@ -1,2 +1,2 @@
 #Top level prints in modules are bad
-print ("Side effect on import")
+print ("Side effect on import") # $ Alert[py/print-during-import]

python/ql/test/2/query-tests/Statements/statements_test.py

@@ -2,7 +2,7 @@
 
 
 def exec_used(val):
-    exec (val)
+    exec (val) # $ Alert[py/use-of-exec]
 
 #Top level print
 import module
@@ -18,7 +18,7 @@ def f(x):
         s = u"Hello World"
     else:
         s = [ u'Hello', u'World']
-    for thing in s:
+    for thing in s: # $ Alert[py/iteration-string-and-sequence]
         print (thing)
 
 import fake_six

python/ql/test/2/query-tests/Summary/LinesOfCode.qlref

@@ -1 +1 @@
-Summary/LinesOfCode.ql
+query: Summary/LinesOfCode.ql

python/ql/test/2/query-tests/Summary/LinesOfUserCode.qlref

@@ -1 +1 @@
-Summary/LinesOfUserCode.ql
+query: Summary/LinesOfUserCode.ql

python/ql/test/2/query-tests/Variables/LeakyComp/LeakyComp.qlref

@@ -1 +1,2 @@
-Variables/LeakingListComprehension.ql
+query: Variables/LeakingListComprehension.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Variables/LeakyComp/test.py

@@ -2,12 +2,12 @@ from __future__ import print_function
 
 def undefined_in_3():
     [x for x in range(3)]
-    print(x)
+    print(x) # $ Alert
 
 def different_in_3():
     y = 10
     [y for y in range(3)]
-    print(y)
+    print(y) # $ Alert
 
 def ok():
     [z for z in range(4)]

python/ql/test/2/query-tests/Variables/undefined/UndefinedExport.py

@@ -1,6 +1,6 @@
 
 
-__all__ = [ "x", "y", "z", "module" ]
+__all__ = [ "x", "y", "z", "module" ] # $ Alert[py/undefined-export]
 
 x = 1
 if 0:

python/ql/test/2/query-tests/Variables/undefined/UndefinedExport.qlref

@@ -1 +1,2 @@
-Variables/UndefinedExport.ql
+query: Variables/UndefinedExport.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Variables/undefined/UndefinedGlobal.qlref

@@ -1 +1,2 @@
-Variables/UndefinedGlobal.ql
+query: Variables/UndefinedGlobal.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Variables/undefined/UninitializedLocal.qlref

@@ -1 +1,2 @@
-Variables/UninitializedLocal.ql
+query: Variables/UninitializedLocal.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/2/query-tests/Variables/undefined/package/__init__.py

@@ -1 +1 @@
-__all__ = [ "module", "not_exists" ]
+__all__ = [ "module", "not_exists" ] # $ Alert[py/undefined-export]

python/ql/test/3/query-tests/Classes/equals-attr/DefineEqualsWhenAddingAttributes.qlref

@@ -1 +1,2 @@
-Classes/DefineEqualsWhenAddingAttributes.ql
+query: Classes/DefineEqualsWhenAddingAttributes.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/3/query-tests/Classes/equals-attr/test.py

@@ -9,7 +9,7 @@ class RedefineEquals:
     def __eq__(self, other):
         return other is "Tuesday"
 
-class C(RedefineEquals):
+class C(RedefineEquals): # $ Alert
 
     def __init__(self, args):
         self.a, self.b = args

python/ql/test/3/query-tests/Classes/inconsistent-mro/InconsistentMRO.qlref

@@ -1 +1,2 @@
-Classes/InconsistentMRO.ql
+query: Classes/InconsistentMRO.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/3/query-tests/Classes/inconsistent-mro/inconsistent_mro.py

@@ -6,12 +6,12 @@ class X(object):
 class Y(X):
     pass
 
-class Z(X, Y):
+class Z(X, Y): # $ Alert
     pass
     
 class O:
     pass
     
 #This is OK in Python 2
-class N(object, O):
+class N(object, O): # $ Alert
     pass

python/ql/test/3/query-tests/Classes/undefined-attribute/MaybeUndefinedClassAttribute.qlref

@@ -1 +1,2 @@
-Classes/MaybeUndefinedClassAttribute.ql
+query: Classes/MaybeUndefinedClassAttribute.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/3/query-tests/Classes/undefined-attribute/UndefinedClassAttribute.qlref

@@ -1 +1,2 @@
-Classes/UndefinedClassAttribute.ql
+query: Classes/UndefinedClassAttribute.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/3/query-tests/Expressions/Arguments/WrongNameForArgumentInCall.qlref

@@ -1 +1,2 @@
-Expressions/WrongNameForArgumentInCall.ql
+query: Expressions/WrongNameForArgumentInCall.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/3/query-tests/Expressions/Arguments/WrongNumberArgumentsInCall.qlref

@@ -1 +1,2 @@
-Expressions/WrongNumberArgumentsInCall.ql
+query: Expressions/WrongNumberArgumentsInCall.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/3/query-tests/Expressions/Arguments/wrong_arguments.py

@@ -9,8 +9,8 @@ f(1, 2, 3, kw1=1)
 f(1, 2, kw1=1, kw2=2)
 
 #Not OK
-f(1, 2, 3, kw1=1, kw3=3)
-f(1, 2, 3, kw3=3)
+f(1, 2, 3, kw1=1, kw3=3) # $ Alert[py/call/wrong-named-argument]
+f(1, 2, 3, kw3=3) # $ Alert[py/call/wrong-named-argument]
 
 
 #ODASA-5897
@@ -21,4 +21,4 @@ def ok():
     return analyze_member_access(msg, original=original, chk=chk)
 
 def bad():
-    return analyze_member_access(msg, original, chk=chk)
+    return analyze_member_access(msg, original, chk=chk) # $ Alert[py/call/wrong-arguments]

python/ql/test/3/query-tests/Expressions/Formatting/WrongNumberArgumentsForFormat.qlref

@@ -1 +1,2 @@
-Expressions/WrongNumberArgumentsForFormat.ql
+query: Expressions/WrongNumberArgumentsForFormat.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/3/query-tests/Expressions/TruncatedDivision/TruncatedDivision.qlref

@@ -1 +1,2 @@
-Expressions/TruncatedDivision.ql
+query: Expressions/TruncatedDivision.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/3/query-tests/Expressions/UseofApply/UseofApply.qlref

@@ -1 +1,2 @@
-Expressions/UseofApply.ql
+query: Expressions/UseofApply.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/3/query-tests/Imports/encoding_error/EncodingError.qlref

@@ -1 +1,2 @@
-Imports/EncodingError.ql
+query: Imports/EncodingError.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/3/query-tests/Imports/syntax_error/EncodingError.qlref

@@ -1 +1,2 @@
-Imports/EncodingError.ql
+query: Imports/EncodingError.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/3/query-tests/Imports/syntax_error/SyntaxError.qlref

@@ -1 +1,2 @@
-Imports/SyntaxError.ql
+query: Imports/SyntaxError.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/3/query-tests/Imports/syntax_error/bad_encoding.py

@@ -8,5 +8,5 @@
 # encoding:shift-jis
 
 def f():
-    print "Python <20>̊J<CC8A><4A><EFBFBD>́A1990 <20>N<EFBFBD><4E><EFBFBD>납<EFBFBD><EB82A9><EFBFBD>J<EFBFBD>n<EFBFBD><6E><EFBFBD><EFBFBD><EFBFBD>Ă<EFBFBD><C482>܂<EFBFBD>"
+    print "Python <20>̊J<CC8A><4A><EFBFBD>́A1990 <20>N<EFBFBD><4E><EFBFBD>납<EFBFBD><EB82A9><EFBFBD>J<EFBFBD>n<EFBFBD><6E><EFBFBD><EFBFBD><EFBFBD>Ă<EFBFBD><C482>܂<EFBFBD>" # $ Alert[py/encoding-error]
 """

python/ql/test/3/query-tests/Imports/syntax_error/nonsense.py

@@ -1,4 +1,4 @@
- `Twas brillig, and the slithy toves
+ `Twas brillig, and the slithy toves # $ Alert[py/syntax-error]
   Did gyre and gimble in the wabe:
 All mimsy were the borogoves,
   And the mome raths outgrabe.

python/ql/test/3/query-tests/Statements/general/ExecUsed.qlref

@@ -1 +1,2 @@
-Statements/ExecUsed.ql
+query: Statements/ExecUsed.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/3/query-tests/Statements/general/TopLevelPrint.qlref

@@ -1 +1,2 @@
-Statements/TopLevelPrint.ql
+query: Statements/TopLevelPrint.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/3/query-tests/Statements/general/module.py

@@ -1,2 +1,2 @@
 #Top level prints in modules are bad
-print ("Side effect on import")
+print ("Side effect on import") # $ Alert[py/print-during-import]

python/ql/test/3/query-tests/Statements/general/statements_test.py

@@ -2,7 +2,7 @@
 
 
 def exec_used(val):
-    exec(val)
+    exec(val) # $ Alert[py/use-of-exec]
 
 #Top level print
 import module

python/ql/test/3/query-tests/Statements/iter/IterableStringOrSequence.qlref

@@ -1 +1,2 @@
-Statements/IterableStringOrSequence.ql
+query: Statements/IterableStringOrSequence.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/3/query-tests/Statements/iter/NonIteratorInForLoop.qlref

@@ -1 +1,2 @@
-Statements/NonIteratorInForLoop.ql
+query: Statements/NonIteratorInForLoop.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/3/query-tests/Statements/iter/async_iterator.py

@@ -23,5 +23,5 @@ async def good():
         yield x
 
 async def bad():
-    async for x in MissingAiter():
+    async for x in MissingAiter(): # $ Alert[py/non-iterable-in-for-loop]
         yield x

python/ql/test/3/query-tests/Statements/iter/statements_test.py

@@ -18,7 +18,7 @@ def f(x):
         s = u"Hello World"
     else:
         s = [ u'Hello', u'World']
-    for thing in s:
+    for thing in s: # $ Alert[py/iteration-string-and-sequence]
         print (thing)
 
 
@@ -31,7 +31,7 @@ class Color(Enum):
 def colors():
     for color in Color:
         print(color)
-    for color in 1:
+    for color in 1: # $ Alert[py/non-iterable-in-for-loop]
         print(color)
 
 colors()

python/ql/test/3/query-tests/Statements/unreachable/UnreachableCode.qlref

@@ -1 +1,2 @@
-Statements/UnreachableCode.ql
+query: Statements/UnreachableCode.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/3/query-tests/Statements/unreachable_suppressed/UnreachableCode.qlref

@@ -1 +1,2 @@
-Statements/UnreachableCode.ql
+query: Statements/UnreachableCode.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/3/query-tests/Summary/LinesOfCode.qlref

@@ -1 +1 @@
-Summary/LinesOfCode.ql
+query: Summary/LinesOfCode.ql

python/ql/test/3/query-tests/Summary/LinesOfUserCode.qlref

@@ -1 +1 @@
-Summary/LinesOfUserCode.ql
+query: Summary/LinesOfUserCode.ql

python/ql/test/3/query-tests/Variables/undefined/UndefinedExport.py

@@ -1,6 +1,6 @@
 
 
-__all__ = [ "x", "y", "z", "module", "w" ]
+__all__ = [ "x", "y", "z", "module", "w" ] # $ Alert[py/undefined-export]
 
 x = 1
 if 0:

python/ql/test/3/query-tests/Variables/undefined/UndefinedExport.qlref

@@ -1 +1,2 @@
-Variables/UndefinedExport.ql
+query: Variables/UndefinedExport.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/3/query-tests/Variables/undefined/UninitializedLocal.qlref

@@ -1 +1,2 @@
-Variables/UninitializedLocal.ql
+query: Variables/UninitializedLocal.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/3/query-tests/Variables/undefined/enum_convert.py

@@ -5,4 +5,4 @@ IntEnum._convert(
         __name__,
         lambda C: C.isupper() and C.startswith('AF_'))
 
-__all__ = [ "Maybe", "Maybe_not" ]
+__all__ = [ "Maybe", "Maybe_not" ] # $ Alert[py/undefined-export]

python/ql/test/3/query-tests/Variables/undefined/package/__init__.py

@@ -1 +1 @@
-__all__ = [ "module", "not_exists" ]
+__all__ = [ "module", "not_exists" ] # $ Alert[py/undefined-export]

python/ql/test/experimental/library-tests/CallGraph-implicit-init/InlineCallGraphTest.qlref

@@ -1 +1 @@
-../CallGraph/InlineCallGraphTest.ql
+query: ../CallGraph/InlineCallGraphTest.ql

python/ql/test/experimental/library-tests/CallGraph-imports/InlineCallGraphTest.qlref

@@ -1 +1 @@
-../CallGraph/InlineCallGraphTest.ql
+query: ../CallGraph/InlineCallGraphTest.ql

python/ql/test/experimental/library-tests/CallGraph-type-annotations/InlineCallGraphTest.qlref

@@ -1 +1 @@
-../CallGraph/InlineCallGraphTest.ql
+query: ../CallGraph/InlineCallGraphTest.ql

python/ql/test/experimental/library-tests/FindSubclass/Find.qlref

@@ -1 +1 @@
-meta/ClassHierarchy/Find.ql
+query: meta/ClassHierarchy/Find.ql

python/ql/test/experimental/query-tests/Classes/Naming/NamingConventionsClasses.py

@@ -1,5 +1,5 @@
 # BAD, do not start class or interface name with lowercase letter
-class badName:
+class badName: # $ Alert
 
     def hello(self):
         print("hello")

python/ql/test/experimental/query-tests/Classes/Naming/NamingConventionsClasses.qlref

@@ -1 +1,2 @@
-experimental/Classes/NamingConventionsClasses.ql
+query: experimental/Classes/NamingConventionsClasses.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/experimental/query-tests/Functions/general/NamingConventionsFunctions.py

@@ -1,7 +1,7 @@
 class Test:
 
     # BAD, do not start function name with uppercase letter
-    def HelloWorld(self):
+    def HelloWorld(self): # $ Alert
         print("hello world")
 
     # GOOD, function name starts with lowercase letter

python/ql/test/experimental/query-tests/Functions/general/NamingConventionsFunctions.qlref

@@ -1 +1,2 @@
-experimental/Functions/NamingConventionsFunctions.ql
+query: experimental/Functions/NamingConventionsFunctions.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/experimental/query-tests/Security/CWE-022-TarSlip/TarSlip.qlref

@@ -1 +1,2 @@
-experimental/Security/CWE-022bis/TarSlipImprov.ql
+query: experimental/Security/CWE-022bis/TarSlipImprov.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/experimental/query-tests/Security/CWE-022-TarSlip/TarSlipImprov.py

@@ -12,14 +12,14 @@ import os.path
 unsafe_filename_tar = sys.argv[2]
 safe_filename_tar = "safe_path.tar"
 
-tar = tarfile.open(unsafe_filename_tar)
+tar = tarfile.open(unsafe_filename_tar) # $ Source[py/tarslip-extended]
 result = []
 for member in tar:
     if ".." in member.name:
         raise ValueError("Path in member name !!!")    
     result.append(member)
 path = unsafe_filename_tar
-tar.extractall(path=path, members=result)
+tar.extractall(path=path, members=result) # $ Alert[py/tarslip-extended]
 tar.close()
 
 
@@ -35,27 +35,27 @@ def members_filter1(tarfile):
         result.append(member)
     return result
 
-tar = tarfile.open(unsafe_filename_tar)
-tar.extractall(path=tempfile.mkdtemp(), members=members_filter1(tar))
+tar = tarfile.open(unsafe_filename_tar) # $ Source[py/tarslip-extended]
+tar.extractall(path=tempfile.mkdtemp(), members=members_filter1(tar)) # $ Alert[py/tarslip-extended]
 tar.close()
 
 
-with tarfile.open(unsafe_filename_tar) as tar:
+with tarfile.open(unsafe_filename_tar) as tar: # $ Source[py/tarslip-extended]
     for entry in tar:
         if ".." in entry.name:
             raise ValueError("Illegal tar archive entry")
-        tar.extract(entry, "/tmp/unpack/")
+        tar.extract(entry, "/tmp/unpack/") # $ Alert[py/tarslip-extended]
 
 
 def _validate_archive_name(name, target):
     if not os.path.abspath(os.path.join(target, name)).startswith(target + os.path.sep):
         raise ValueError(f"Provided language pack contains invalid name {name}")
 
-with tarfile.open(unsafe_filename_tar) as tar:
+with tarfile.open(unsafe_filename_tar) as tar: # $ Source[py/tarslip-extended]
     target = "/tmp/unpack"
     for entry in tar:
         _validate_archive_name(entry.name, target)
-        tar.extract(entry, target)
+        tar.extract(entry, target) # $ Alert[py/tarslip-extended]
 
 
 def members_filter2(tarfile):
@@ -85,10 +85,10 @@ def _validate_archive_name(name, target):
         raise ValueError(f"Provided language pack contains invalid name {name}")
 
 target = "/tmp/unpack"
-with tarfile.open(unsafe_filename_tar, "r") as tar:
+with tarfile.open(unsafe_filename_tar, "r") as tar: # $ Source[py/tarslip-extended]
     for info in tar.getmembers():
         _validate_tar_info(info, target)
-    tar.extractall(target)
+    tar.extractall(target) # $ Alert[py/tarslip-extended]
 
 
 def members_filter3(tarfile):
@@ -108,11 +108,11 @@ tar.extractall(path=tempfile.mkdtemp(), members=members_filter3(tar))
 tar.close()
 
     
-tar = tarfile.open(unsafe_filename_tar)
+tar = tarfile.open(unsafe_filename_tar) # $ Source[py/tarslip-extended]
 tarf = tar.getmembers()
 for f in tarf:
     if not f.issym():
-        tar.extractall(path=tempfile.mkdtemp(), members=[f])
+        tar.extractall(path=tempfile.mkdtemp(), members=[f]) # $ Alert[py/tarslip-extended]
 tar.close()
 
 
@@ -120,27 +120,27 @@ class MKTar(TarFile):
     pass
 
 tarball = unsafe_filename_tar
-with MKTar.open(name=tarball) as tar:
+with MKTar.open(name=tarball) as tar: # $ Source[py/tarslip-extended]
     for entry in tar:
-        tar._extract_member(entry, entry.name)
+        tar._extract_member(entry, entry.name) # $ Alert[py/tarslip-extended]
 
 
 tarball = unsafe_filename_tar
-with tarfile.open(tarball) as tar:
-    tar.extractall()
+with tarfile.open(tarball) as tar: # $ Source[py/tarslip-extended]
+    tar.extractall() # $ Alert[py/tarslip-extended]
 
 
-tar = tarfile.open(unsafe_filename_tar)
-tar.extractall(path=tempfile.mkdtemp(), members=None)
+tar = tarfile.open(unsafe_filename_tar) # $ Source[py/tarslip-extended]
+tar.extractall(path=tempfile.mkdtemp(), members=None) # $ Alert[py/tarslip-extended]
 
 
 class MKTar(tarfile.TarFile):
     pass
 
 tarball = unsafe_filename_tar
-with MKTar.open(name=tarball) as tar:
+with MKTar.open(name=tarball) as tar: # $ Source[py/tarslip-extended]
     for entry in tar:
-        tar._extract_member(entry, entry.name)
+        tar._extract_member(entry, entry.name) # $ Alert[py/tarslip-extended]
 
 
 @contextmanager
@@ -148,7 +148,7 @@ def py2_tarxz(filename):
     with tempfile.TemporaryFile() as tmp:
         subprocess.check_call(["xz", "-dc", filename], stdout=tmp.fileno())
         tmp.seek(0)
-        with closing(tarfile.TarFile(fileobj=tmp)) as tf:
+        with closing(tarfile.TarFile(fileobj=tmp)) as tf: # $ Source[py/tarslip-extended]
             yield tf
 
 def unpack_tarball(tar_filename, dest):     
@@ -156,7 +156,7 @@ def unpack_tarball(tar_filename, dest):
         # Py 2.7 lacks lzma support
         tar_cm = py2_tarxz(tar_filename)
     else:
-        tar_cm = closing(tarfile.open(tar_filename))
+        tar_cm = closing(tarfile.open(tar_filename)) # $ Source[py/tarslip-extended]
 
     base_dir = None
     with tar_cm as tarc:
@@ -166,32 +166,32 @@ def unpack_tarball(tar_filename, dest):
                 base_dir = base_name
             elif base_dir != base_name:
                 print('Unexpected path in %s: %s' % (tar_filename, base_name))
-        tarc.extractall(dest)
+        tarc.extractall(dest) # $ Alert[py/tarslip-extended]
     return os.path.join(dest, base_dir)
 
 unpack_tarball(unsafe_filename_tar, "/tmp/unpack")
 
 
 tarball = unsafe_filename_tar
-with tarfile.open(name=tarball) as tar:
+with tarfile.open(name=tarball) as tar: # $ Source[py/tarslip-extended]
     for entry in tar:
-        tar._extract_member(entry, entry.name)
+        tar._extract_member(entry, entry.name) # $ Alert[py/tarslip-extended]
 
 
 tarball = unsafe_filename_tar
-with tarfile.open(name=tarball) as tar:
+with tarfile.open(name=tarball) as tar: # $ Source[py/tarslip-extended]
     for entry in tar:
-        tar.extract(entry, "/tmp/unpack/")
+        tar.extract(entry, "/tmp/unpack/") # $ Alert[py/tarslip-extended]
 
 
 tarball = unsafe_filename_tar
-tar = tarfile.open(tarball)
-tar.extractall("/tmp/unpack/")
+tar = tarfile.open(tarball) # $ Source[py/tarslip-extended]
+tar.extractall("/tmp/unpack/") # $ Alert[py/tarslip-extended]
 
 
 tarball = unsafe_filename_tar
-with tarfile.open(tarball, "r") as tar:
-    tar.extractall(path="/tmp/unpack/", members=tar)
+with tarfile.open(tarball, "r") as tar: # $ Source[py/tarslip-extended]
+    tar.extractall(path="/tmp/unpack/", members=tar) # $ Alert[py/tarslip-extended]
 
 
 def members_filter4(tarfile):
@@ -207,8 +207,8 @@ tar.extractall(path=tempfile.mkdtemp(), members=members_filter4(tar))
 tar.close()
 
 
-with tarfile.open(unsafe_filename_tar, "r") as tar:
-    tar.extractall(path="/tmp/unpack")
+with tarfile.open(unsafe_filename_tar, "r") as tar: # $ Source[py/tarslip-extended]
+    tar.extractall(path="/tmp/unpack") # $ Alert[py/tarslip-extended]
 
 
 def members_filter5(tarfile):
@@ -228,12 +228,12 @@ filename = unsafe_filename_tar
 tmp_dir = "/tmp/"
 
 read_type = "r:gz" if filename.endswith("tgz") else "r"
-with tarfile.open(filename, read_type) as corpus_tar:
+with tarfile.open(filename, read_type) as corpus_tar: # $ Source[py/tarslip-extended]
     members = []
     for f in corpus_tar:
         if not os.path.isfile(os.path.join(tmp_dir, f.name)):
             members.append(f)
-    corpus_tar.extractall(tmp_dir, members=members)
+    corpus_tar.extractall(tmp_dir, members=members) # $ Alert[py/tarslip-extended]
 
 
 def members_filter6(tarfile):
@@ -251,66 +251,66 @@ tar.close()
 
 archive_path = unsafe_filename_tar
 target_dir = "/tmp/unpack"
-tarfile.open(archive_path, "r").extractall(path=target_dir)
+tarfile.open(archive_path, "r").extractall(path=target_dir) # $ Alert[py/tarslip-extended]
 
 
 tarball = unsafe_filename_tar
-with tarfile.open(tarball) as tar:
+with tarfile.open(tarball) as tar: # $ Source[py/tarslip-extended]
     for entry in tar:
         if entry.isfile():
-            tar.extract(entry, "/tmp/unpack/")
+            tar.extract(entry, "/tmp/unpack/") # $ Alert[py/tarslip-extended]
 
 
-with tarfile.open(unsafe_filename_tar) as tar:
+with tarfile.open(unsafe_filename_tar) as tar: # $ Source[py/tarslip-extended]
     for entry in tar:
         if entry.name.startswith("/"):
             raise ValueError("Illegal tar archive entry")
-        tar.extract(entry, "/tmp/unpack/")
+        tar.extract(entry, "/tmp/unpack/") # $ Alert[py/tarslip-extended]
 
 tarball = unsafe_filename_tar
-with tarfile.TarFile(tarball, mode="r") as tar:
+with tarfile.TarFile(tarball, mode="r") as tar: # $ Source[py/tarslip-extended]
     for entry in tar:
         if entry.isfile():
-            tar.extract(entry, "/tmp/unpack/")
+            tar.extract(entry, "/tmp/unpack/") # $ Alert[py/tarslip-extended]
 
-with tarfile.open(unsafe_filename_tar) as tar:
+with tarfile.open(unsafe_filename_tar) as tar: # $ Source[py/tarslip-extended]
     for entry in tar:
         if os.path.isabs(entry.name):
             raise ValueError("Illegal tar archive entry")
-        tar.extract(entry, "/tmp/unpack/")
+        tar.extract(entry, "/tmp/unpack/") # $ Alert[py/tarslip-extended]
 
 
-with tarfile.TarFile(unsafe_filename_tar, mode="r") as tar:
-    tar.extractall(path="/tmp/unpack")
+with tarfile.TarFile(unsafe_filename_tar, mode="r") as tar: # $ Source[py/tarslip-extended]
+    tar.extractall(path="/tmp/unpack") # $ Alert[py/tarslip-extended]
 
 
-tar = tarfile.open(filename)
-tar.extractall(path=tempfile.mkdtemp(), members=tar.getmembers())
+tar = tarfile.open(filename) # $ Source[py/tarslip-extended]
+tar.extractall(path=tempfile.mkdtemp(), members=tar.getmembers()) # $ Alert[py/tarslip-extended]
 tar.close()
 
 
-tar = tarfile.open(unsafe_filename_tar)
-tar.extractall(path=tempfile.mkdtemp(), members=None)
+tar = tarfile.open(unsafe_filename_tar) # $ Source[py/tarslip-extended]
+tar.extractall(path=tempfile.mkdtemp(), members=None) # $ Alert[py/tarslip-extended]
 
 
 tar.extractall(path=tempfile.mkdtemp(), members=members_filter4(tar))
 tar.close()
 
 
-with tarfile.TarFile(unsafe_filename_tar, mode="r") as tar:
-    tar.extractall(path="/tmp/unpack/", members=tar)
+with tarfile.TarFile(unsafe_filename_tar, mode="r") as tar: # $ Source[py/tarslip-extended]
+    tar.extractall(path="/tmp/unpack/", members=tar) # $ Alert[py/tarslip-extended]
 
 
-tar = tarfile.open(unsafe_filename_tar)
+tar = tarfile.open(unsafe_filename_tar) # $ Source[py/tarslip-extended]
 result = []
 for member in tar:
     if member.issym():
         raise ValueError("But it is a symlink")   
     result.append(member)
-tar.extractall(path=tempfile.mkdtemp(), members=result)
+tar.extractall(path=tempfile.mkdtemp(), members=result) # $ Alert[py/tarslip-extended]
 tar.close()
 
 
 archive_path = unsafe_filename_tar
 target_dir = "/tmp/unpack"
-tarfile.TarFile(unsafe_filename_tar, mode="r").extractall(path=target_dir)
+tarfile.TarFile(unsafe_filename_tar, mode="r").extractall(path=target_dir) # $ Alert[py/tarslip-extended]

python/ql/test/experimental/query-tests/Security/CWE-022-TarSlip/ZipSlip.qlref

@@ -1 +1,2 @@
-experimental/Security/CWE-022/ZipSlip.ql
+query: experimental/Security/CWE-022/ZipSlip.ql
+postprocess: utils/test/InlineExpectationsTestQuery.ql

python/ql/test/experimental/query-tests/Security/CWE-022-TarSlip/zipslip_bad.py

@@ -5,35 +5,35 @@ import gzip
 import zipfile
 
 def unzip(filename):
-    with tarfile.open(filename) as zipf:
+    with tarfile.open(filename) as zipf: # $ Alert[py/zipslip]
     #BAD : This could write any file on the filesystem.
         for entry in zipf:
-            shutil.move(entry, "/tmp/unpack/")
+            shutil.move(entry, "/tmp/unpack/") # $ Sink[py/zipslip]
 
 def unzip1(filename):
-    with gzip.open(filename) as zipf:
+    with gzip.open(filename) as zipf: # $ Alert[py/zipslip]
     #BAD : This could write any file on the filesystem.
         for entry in zipf:
-            shutil.copy2(entry, "/tmp/unpack/")
+            shutil.copy2(entry, "/tmp/unpack/") # $ Sink[py/zipslip]
 
 def unzip2(filename):
-    with bz2.open(filename) as zipf:
+    with bz2.open(filename) as zipf: # $ Alert[py/zipslip]
     #BAD : This could write any file on the filesystem.
         for entry in zipf:
-            shutil.copyfile(entry, "/tmp/unpack/")
+            shutil.copyfile(entry, "/tmp/unpack/") # $ Sink[py/zipslip]
 
 def unzip3(filename):
     zf = zipfile.ZipFile(filename)
-    with zf.namelist() as filelist:
+    with zf.namelist() as filelist: # $ Alert[py/zipslip]
     #BAD : This could write any file on the filesystem.
         for x in filelist:
-            shutil.copy(x, "/tmp/unpack/")
+            shutil.copy(x, "/tmp/unpack/") # $ Sink[py/zipslip]
 
 def unzip4(filename):
     zf = zipfile.ZipFile(filename)
-    filelist = zf.namelist()
+    filelist = zf.namelist() # $ Alert[py/zipslip]
     for x in filelist:
         with zf.open(x) as srcf:
-            shutil.copyfileobj(x, "/tmp/unpack/")
+            shutil.copyfileobj(x, "/tmp/unpack/") # $ Sink[py/zipslip]
 
 import tty # to set the import root so we can identify the standard library