Merge pull request #15070 from tamasvajk/standalone/exclusions

C#: Remove unneeded options and add support for `paths/paths-ignore` in standalone

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

@@ -590,7 +590,7 @@ namespace Semmle.Autobuild.CSharp.Tests
         [Fact]
         public void TestLinuxBuildlessExtractionSolution()
         {
-            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone foo.sln"] = 0;
+            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
             actions.FileExists["csharp.log"] = true;
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
@@ -598,7 +598,7 @@ namespace Semmle.Autobuild.CSharp.Tests
             actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
             actions.EnumerateDirectories[@"C:\Project"] = "";
 
-            var autobuilder = CreateAutoBuilder(false, buildless: "true", solution: "foo.sln");
+            var autobuilder = CreateAutoBuilder(false, buildless: "true");
             TestAutobuilderScript(autobuilder, 0, 1);
         }
 
@@ -890,7 +890,7 @@ namespace Semmle.Autobuild.CSharp.Tests
         [Fact]
         public void TestSkipNugetBuildless()
         {
-            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone foo.sln --skip-nuget"] = 0;
+            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
             actions.FileExists["csharp.log"] = true;
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
@@ -898,7 +898,7 @@ namespace Semmle.Autobuild.CSharp.Tests
             actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
             actions.EnumerateDirectories[@"C:\Project"] = "";
 
-            var autobuilder = CreateAutoBuilder(false, buildless: "true", solution: "foo.sln", nugetRestore: "false");
+            var autobuilder = CreateAutoBuilder(false, buildless: "true");
             TestAutobuilderScript(autobuilder, 0, 1);
         }
 

csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs

@@ -17,7 +17,7 @@ namespace Semmle.Autobuild.CSharp
 
         public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
         {
-            BuildScript GetCommand(string? solution)
+            BuildScript GetCommand()
             {
                 string standalone;
                 if (builder.CodeQLExtractorLangRoot is not null && builder.CodeQlPlatform is not null)
@@ -32,14 +32,6 @@ namespace Semmle.Autobuild.CSharp
                 var cmd = new CommandBuilder(builder.Actions);
                 cmd.RunCommand(standalone);
 
-                if (solution is not null)
-                    cmd.QuoteArgument(solution);
-
-                if (!builder.Options.NugetRestore)
-                {
-                    cmd.Argument("--skip-nuget");
-                }
-
                 if (!string.IsNullOrEmpty(this.dotNetPath))
                 {
                     cmd.Argument("--dotnet");
@@ -50,16 +42,11 @@ namespace Semmle.Autobuild.CSharp
             }
 
             if (!builder.Options.Buildless)
+            {
                 return BuildScript.Failure;
+            }
 
-            if (!builder.Options.Solution.Any())
-                return GetCommand(null);
-
-            var script = BuildScript.Success;
-            foreach (var solution in builder.Options.Solution)
-                script &= GetCommand(solution);
-
-            return script;
+            return GetCommand();
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -80,67 +80,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             this.nonGeneratedSources = allNonBinaryFiles.SelectFileNamesByExtension(".cs").ToList();
             this.generatedSources = new();
             var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj");
-            var solutions = options.SolutionFile is not null
-                ? new[] { options.SolutionFile }
-                : allNonBinaryFiles.SelectFileNamesByExtension(".sln");
-            var dllPaths = options.DllDirs.Count == 0
-                ? allFiles.SelectFileNamesByExtension(".dll").ToHashSet()
-                : options.DllDirs.Select(Path.GetFullPath).ToHashSet();
-
-            if (options.UseNuGet)
-            {
-                try
-                {
-                    var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, progressMonitor);
-                    nuget.InstallPackages();
-
-                    var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
-                    var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet();
-                    var excludedPaths = nugetPackageDllPaths
-                        .Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools"));
-
-                    foreach (var excludedPath in excludedPaths)
-                    {
-                        progressMonitor.LogInfo($"Excluded Nuget DLL: {excludedPath}");
-                    }
-
-                    nugetPackageDllPaths.ExceptWith(excludedPaths);
-                    dllPaths.UnionWith(nugetPackageDllPaths);
-                }
-                catch (FileNotFoundException)
-                {
-                    progressMonitor.MissingNuGet();
-                }
-
-                var restoredProjects = RestoreSolutions(solutions, out var assets1);
-                var projects = allProjects.Except(restoredProjects);
-                RestoreProjects(projects, out var assets2);
-
-                var dependencies = Assets.GetCompilationDependencies(progressMonitor, assets1.Union(assets2));
-
-                var paths = dependencies
-                    .Paths
-                    .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d))
-                    .ToList();
-                dllPaths.UnionWith(paths);
-
-                LogAllUnusedPackages(dependencies);
-                DownloadMissingPackages(allNonBinaryFiles, dllPaths);
-            }
-
-            var frameworkLocations = new HashSet<string>();
+            var allSolutions = allNonBinaryFiles.SelectFileNamesByExtension(".sln");
+            var dllPaths = allFiles.SelectFileNamesByExtension(".dll").ToHashSet();
 
+            RestoreNugetPackages(allNonBinaryFiles, allProjects, allSolutions, dllPaths);
             // Find DLLs in the .Net / Asp.Net Framework
-            // This block needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies.
-            if (options.ScanNetFrameworkDlls)
-            {
-                AddNetFrameworkDlls(dllPaths, frameworkLocations);
-                AddAspNetCoreFrameworkDlls(dllPaths, frameworkLocations);
-                AddMicrosoftWindowsDesktopDlls(dllPaths, frameworkLocations);
-            }
+            // This needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies.
+            var frameworkLocations = AddFrameworkDlls(dllPaths);
 
             assemblyCache = new AssemblyCache(dllPaths, frameworkLocations, progressMonitor);
-            AnalyseSolutions(solutions);
+            AnalyseSolutions(allSolutions);
 
             foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename))
             {
@@ -182,6 +131,58 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 DateTime.Now - startTime);
         }
 
+        private HashSet<string> AddFrameworkDlls(HashSet<string> dllPaths)
+        {
+            var frameworkLocations = new HashSet<string>();
+
+            AddNetFrameworkDlls(dllPaths, frameworkLocations);
+            AddAspNetCoreFrameworkDlls(dllPaths, frameworkLocations);
+            AddMicrosoftWindowsDesktopDlls(dllPaths, frameworkLocations);
+
+            return frameworkLocations;
+        }
+
+        private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<string> allProjects, IEnumerable<string> allSolutions, HashSet<string> dllPaths)
+        {
+            try
+            {
+                var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, progressMonitor);
+                nuget.InstallPackages();
+
+                var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
+                var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet();
+                var excludedPaths = nugetPackageDllPaths
+                    .Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools"));
+
+                foreach (var excludedPath in excludedPaths)
+                {
+                    progressMonitor.LogInfo($"Excluded Nuget DLL: {excludedPath}");
+                }
+
+                nugetPackageDllPaths.ExceptWith(excludedPaths);
+                dllPaths.UnionWith(nugetPackageDllPaths);
+            }
+            catch (FileNotFoundException)
+            {
+                progressMonitor.MissingNuGet();
+            }
+
+            var restoredProjects = RestoreSolutions(allSolutions, out var assets1);
+            var projects = allProjects.Except(restoredProjects);
+            RestoreProjects(projects, out var assets2);
+
+            var dependencies = Assets.GetCompilationDependencies(progressMonitor, assets1.Union(assets2));
+
+            var paths = dependencies
+                .Paths
+                .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d))
+                .ToList();
+            dllPaths.UnionWith(paths);
+
+            LogAllUnusedPackages(dependencies);
+            DownloadMissingPackages(allNonBinaryFiles, dllPaths);
+        }
+
         private static bool IsPathInSubfolder(string path, string rootFolder, string subFolder)
         {
             return path.IndexOf(
@@ -192,11 +193,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
         private void RemoveNugetAnalyzerReferences()
         {
-            if (!options.UseNuGet)
-            {
-                return;
-            }
-
             var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant();
             if (packageFolder == null)
             {
@@ -279,11 +275,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
             string? runtimeLocation = null;
 
-            if (options.UseSelfContainedDotnet)
-            {
-                runtimeLocation = Runtime.ExecutingRuntime;
-            }
-            else if (fileContent.IsNewProjectStructureUsed)
+            if (fileContent.IsNewProjectStructureUsed)
             {
                 runtimeLocation = Runtime.NetCoreRuntime;
             }
@@ -301,11 +293,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
         private void RemoveNugetPackageReference(string packagePrefix, ISet<string> dllPaths)
         {
-            if (!options.UseNuGet)
-            {
-                return;
-            }
-
             var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant();
             if (packageFolder == null)
             {
@@ -353,11 +340,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
         private string? GetPackageDirectory(string packagePrefix)
         {
-            if (!options.UseNuGet)
-            {
-                return null;
-            }
-
             return new DirectoryInfo(packageDirectory.DirInfo.FullName)
                 .EnumerateDirectories(packagePrefix + "*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
                 .FirstOrDefault()?
@@ -366,11 +348,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
         private IEnumerable<string> GetAllPackageDirectories()
         {
-            if (!options.UseNuGet)
-            {
-                return Enumerable.Empty<string>();
-            }
-
             return new DirectoryInfo(packageDirectory.DirInfo.FullName)
                 .EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
                 .Select(d => d.Name);
@@ -455,14 +432,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
         private IEnumerable<FileInfo> GetAllFiles()
         {
-            var files = sourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true })
-                .Where(d => !options.ExcludesFile(d.FullName));
+            IEnumerable<FileInfo> files = sourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true });
 
             if (options.DotNetPath != null)
             {
                 files = files.Where(f => !f.FullName.StartsWith(options.DotNetPath, StringComparison.OrdinalIgnoreCase));
             }
 
+            files = new FilePathFilter(sourceDir, progressMonitor).Filter(files);
             return files;
         }
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyOptions.cs

@@ -10,43 +10,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     /// </summary>
     public interface IDependencyOptions
     {
-        /// <summary>
-        /// Directories to search DLLs in.
-        /// </summary>
-        IList<string> DllDirs { get; }
-
-        /// <summary>
-        /// Files/patterns to exclude.
-        /// </summary>
-        IList<string> Excludes { get; }
-
-        /// <summary>
-        /// Whether to analyse NuGet packages.
-        /// </summary>
-        bool UseNuGet { get; }
-
-        /// <summary>
-        /// The solution file to analyse, or null if not specified.
-        /// </summary>
-        string? SolutionFile { get; }
-
-        /// <summary>
-        /// Whether to use the packaged dotnet runtime.
-        /// </summary>
-        bool UseSelfContainedDotnet { get; }
-
-        /// <summary>
-        /// Whether to search the .Net framework directory.
-        /// </summary>
-        bool ScanNetFrameworkDlls { get; }
-
-        /// <summary>
-        /// Determine whether the given path should be excluded.
-        /// </summary>
-        /// <param name="path">The path to query.</param>
-        /// <returns>True iff the path matches an exclusion.</returns>
-        bool ExcludesFile(string path);
-
         /// <summary>
         /// The number of threads to use.
         /// </summary>
@@ -62,21 +25,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     {
         public static IDependencyOptions Default => new DependencyOptions();
 
-        public IList<string> DllDirs { get; set; } = new List<string>();
-
-        public IList<string> Excludes { get; set; } = new List<string>();
-
-        public bool UseNuGet { get; set; } = true;
-
-        public string? SolutionFile { get; set; }
-
-        public bool UseSelfContainedDotnet { get; set; } = false;
-
-        public bool ScanNetFrameworkDlls { get; set; } = true;
-
-        public bool ExcludesFile(string path) =>
-            Excludes.Any(path.Contains);
-
         public int Threads { get; set; } = EnvironmentVariables.GetDefaultNumberOfThreads();
 
         public string? DotNetPath { get; set; } = null;

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs

@@ -0,0 +1,99 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Security.Cryptography.X509Certificates;
+using System.Text.RegularExpressions;
+using Semmle.Util;
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    public class FilePathFilter
+    {
+        private readonly string rootFolder;
+        private readonly IProgressMonitor progressMonitor;
+
+        public FilePathFilter(DirectoryInfo sourceDir, IProgressMonitor progressMonitor)
+        {
+            rootFolder = FileUtils.ConvertToUnix(sourceDir.FullName.ToLowerInvariant());
+            this.progressMonitor = progressMonitor;
+        }
+
+        private class FileInclusion(string path, bool include)
+        {
+            public string Path { get; } = path;
+
+            public bool Include { get; set; } = include;
+        }
+
+        private record class PathFilter(Regex Regex, bool Include);
+
+        public IEnumerable<FileInfo> Filter(IEnumerable<FileInfo> files)
+        {
+            var filters = (Environment.GetEnvironmentVariable("LGTM_INDEX_FILTERS") ?? string.Empty).Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
+            if (filters.Length == 0)
+            {
+                return files;
+            }
+
+            var pathFilters = new List<PathFilter>();
+
+            foreach (var filter in filters)
+            {
+                bool include;
+                string filterText;
+                if (filter.StartsWith("include:"))
+                {
+                    include = true;
+                    filterText = filter.Substring("include:".Length);
+                }
+                else if (filter.StartsWith("exclude:"))
+                {
+                    include = false;
+                    filterText = filter.Substring("exclude:".Length);
+                }
+                else
+                {
+                    progressMonitor.Log(Severity.Info, $"Invalid filter: {filter}");
+                    continue;
+                }
+
+                var regex = new FilePattern(filterText).RegexPattern;
+                progressMonitor.Log(Severity.Info, $"Filtering {(include ? "in" : "out")} files matching '{regex}'. Original glob filter: '{filter}'");
+                pathFilters.Add(new PathFilter(new Regex(regex, RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline), include));
+            }
+
+            var includeByDefault = pathFilters.All(f => !f.Include);
+            var unfilteredResult = files.Select(f =>
+                new
+                {
+                    FileInfo = f,
+                    FileInclusion = new FileInclusion(
+                        FileUtils.ConvertToUnix(f.FullName.ToLowerInvariant()).Replace(rootFolder, string.Empty).TrimStart('/'),
+                        includeByDefault)
+                });
+
+            // Move included pathfilters to the front of the list:
+            pathFilters.Sort((pf1, pf2) => -1 * pf1.Include.CompareTo(pf2.Include));
+            return unfilteredResult.Where(f =>
+            {
+                var include = f.FileInclusion.Include;
+                foreach (var pathFilter in pathFilters)
+                {
+                    if (pathFilter.Regex.IsMatch(f.FileInclusion.Path))
+                    {
+                        include = pathFilter.Include;
+                    }
+                }
+
+                if (!include)
+                {
+                    progressMonitor.Log(Severity.Info, $"Excluding '{f.FileInfo.FullName}'");
+                }
+
+                return include;
+            }).Select(f => f.FileInfo);
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IProgressMonitor.cs

@@ -0,0 +1,9 @@
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    public interface IProgressMonitor
+    {
+        void Log(Severity severity, string message);
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs

@@ -3,7 +3,7 @@ using Semmle.Util.Logging;
 
 namespace Semmle.Extraction.CSharp.DependencyFetching
 {
-    internal class ProgressMonitor
+    internal class ProgressMonitor : IProgressMonitor
     {
         private readonly ILogger logger;
 

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs

@@ -147,20 +147,17 @@ namespace Semmle.Extraction.CSharp.Standalone
                 return ExitCode.Errors;
             }
 
-            if (!options.SkipExtraction)
-            {
-                using var fileLogger = CSharp.Extractor.MakeLogger(options.Verbosity, false);
+            using var fileLogger = CSharp.Extractor.MakeLogger(options.Verbosity, false);
 
-                logger.Log(Severity.Info, "");
-                logger.Log(Severity.Info, "Extracting...");
-                ExtractStandalone(
-                    a.Extraction.Sources,
-                    a.References,
-                    new ExtractionProgress(logger),
-                    fileLogger,
-                    options);
-                logger.Log(Severity.Info, $"Extraction completed in {stopwatch.Elapsed}");
-            }
+            logger.Log(Severity.Info, "");
+            logger.Log(Severity.Info, "Extracting...");
+            ExtractStandalone(
+                a.Extraction.Sources,
+                a.References,
+                new ExtractionProgress(logger),
+                fileLogger,
+                options);
+            logger.Log(Severity.Info, $"Extraction completed in {stopwatch.Elapsed}");
 
             return ExitCode.Ok;
         }

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Options.cs

@@ -21,21 +21,6 @@ namespace Semmle.Extraction.CSharp.Standalone
                 case "help":
                     Help = true;
                     return true;
-                case "dry-run":
-                    SkipExtraction = value;
-                    return true;
-                case "skip-nuget":
-                    dependencies.UseNuGet = !value;
-                    return true;
-                case "all-references":
-                    AnalyseCsProjFiles = !value;
-                    return true;
-                case "skip-dotnet":
-                    dependencies.ScanNetFrameworkDlls = !value;
-                    return true;
-                case "self-contained-dotnet":
-                    dependencies.UseSelfContainedDotnet = value;
-                    return true;
                 default:
                     return base.HandleFlag(key, value);
             }
@@ -45,12 +30,6 @@ namespace Semmle.Extraction.CSharp.Standalone
         {
             switch (key)
             {
-                case "exclude":
-                    dependencies.Excludes.Add(value);
-                    return true;
-                case "references":
-                    dependencies.DllDirs.Add(value);
-                    return true;
                 case "dotnet":
                     dependencies.DotNetPath = value;
                     return true;
@@ -61,13 +40,6 @@ namespace Semmle.Extraction.CSharp.Standalone
 
         public override bool HandleArgument(string arg)
         {
-            dependencies.SolutionFile = arg;
-            var fi = new FileInfo(dependencies.SolutionFile);
-            if (!fi.Exists)
-            {
-                System.Console.WriteLine($"[{Environment.CurrentManagedThreadId:D3}] Error: The solution {fi.FullName} does not exist");
-                Errors = true;
-            }
             return true;
         }
 
@@ -80,7 +52,7 @@ namespace Semmle.Extraction.CSharp.Standalone
         /// <summary>
         /// The directory containing the source code;
         /// </summary>
-        public string SrcDir { get; } = System.IO.Directory.GetCurrentDirectory();
+        public string SrcDir { get; } = Directory.GetCurrentDirectory();
 
         private readonly DependencyOptions dependencies = new DependencyOptions();
         /// <summary>
@@ -88,16 +60,6 @@ namespace Semmle.Extraction.CSharp.Standalone
         /// </summary>
         public IDependencyOptions Dependencies => dependencies;
 
-        /// <summary>
-        /// Whether to search .csproj files.
-        /// </summary>
-        public bool AnalyseCsProjFiles { get; private set; } = true;
-
-        /// <summary>
-        /// Whether the extraction phase should be skipped (dry-run).
-        /// </summary>
-        public bool SkipExtraction { get; private set; } = false;
-
         /// <summary>
         /// Whether errors were encountered parsing the arguments.
         /// </summary>
@@ -115,17 +77,9 @@ namespace Semmle.Extraction.CSharp.Standalone
         {
             output.WriteLine("C# standalone extractor\n\nExtracts a C# project in the current directory without performing a build.\n");
             output.WriteLine("Additional options:\n");
-            output.WriteLine("    xxx.sln          Restrict sources to given solution");
-            output.WriteLine("    --exclude:xxx    Exclude a file or directory (can be specified multiple times)");
-            output.WriteLine("    --references:xxx Scan additional files or directories for assemblies (can be specified multiple times)");
-            output.WriteLine("    --skip-dotnet    Do not reference the .Net Framework");
-            output.WriteLine("    --dry-run        Stop before extraction");
-            output.WriteLine("    --skip-nuget     Do not download nuget packages");
-            output.WriteLine("    --all-references Use all references (default is to only use references in .csproj files)");
             output.WriteLine("    --threads:nnn    Specify number of threads (default=CPU cores)");
             output.WriteLine("    --verbose        Produce more output");
             output.WriteLine("    --pdb            Cross-reference information from PDBs where available");
-            output.WriteLine("    --self-contained-dotnet    Use the .Net Framework packaged with the extractor");
         }
 
         private Options()

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs

@@ -29,7 +29,7 @@ namespace Semmle.Extraction.CSharp.Standalone
             dependencyManager = new DependencyManager(options.SrcDir, options.Dependencies, logger);
             References = dependencyManager.ReferenceFiles;
             Extraction = new Extraction(options.SrcDir);
-            Extraction.Sources.AddRange(options.Dependencies.SolutionFile is null ? dependencyManager.AllSourceFiles : dependencyManager.ProjectSourceFiles);
+            Extraction.Sources.AddRange(dependencyManager.AllSourceFiles);
         }
 
         public IEnumerable<string> References { get; }

csharp/extractor/Semmle.Extraction.Tests/FilePathFilter.cs

@@ -0,0 +1,249 @@
+using Xunit;
+using Semmle.Extraction.CSharp.DependencyFetching;
+using Semmle.Util.Logging;
+using System.Linq;
+using System.Collections.Generic;
+using System.IO;
+using System;
+
+namespace Semmle.Extraction.Tests
+{
+    public class FilePathFilterTest
+    {
+        private class ProgressMonitorStub : IProgressMonitor
+        {
+            public List<string> Messages { get; } = [];
+
+            public void Log(Severity severity, string message)
+            {
+                Messages.Add(message);
+            }
+        }
+
+        private static (FilePathFilter TestSubject, ProgressMonitorStub progressMonitor, IEnumerable<FileInfo> Files) TestSetup()
+        {
+            return TestSetup("/a/b",
+            [
+                "/a/b/c/d/e/f.cs",
+                "/a/b/c/d/e/g.cs",
+                "/a/b/c/d/e/h.cs",
+                "/a/b/c/x/y/i.cs",
+                "/a/b/c/x/z/i.cs"
+            ]);
+        }
+
+        private static (FilePathFilter TestSubject, ProgressMonitorStub progressMonitor, IEnumerable<FileInfo> Files) TestSetup(string root, IEnumerable<string> paths)
+        {
+            root = GetPlatformSpecifixPath(root);
+            paths = GetPlatformSpecifixPaths(paths);
+
+            var progressMonitor = new ProgressMonitorStub();
+
+            var filePathFilter = new FilePathFilter(new DirectoryInfo(root), progressMonitor);
+            return (filePathFilter, progressMonitor, paths.Select(p => new FileInfo(p)));
+        }
+
+        private static string GetPlatformSpecifixPath(string file)
+        {
+            return file.Replace('/', Path.DirectorySeparatorChar);
+        }
+
+        private static IEnumerable<string> GetPlatformSpecifixPaths(IEnumerable<string> files)
+        {
+            return files.Select(GetPlatformSpecifixPath);
+        }
+
+        private static IEnumerable<FileInfo> GetExpected(IEnumerable<string> files)
+        {
+            files = GetPlatformSpecifixPaths(files);
+            return files.Select(f => new FileInfo(f));
+        }
+
+        private static void AssertFileInfoEquivalence(IEnumerable<FileInfo>? expected, IEnumerable<FileInfo>? actual)
+        {
+            Assert.Equivalent(expected?.Select(f => f.FullName), actual?.Select(f => f.FullName), strict: true);
+        }
+
+        [Fact]
+        public void TestNoFilter()
+        {
+            (var testSubject, var progressMonitor, var files) = TestSetup();
+
+            Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", null);
+
+            var filtered = testSubject.Filter(files);
+
+            AssertFileInfoEquivalence(files, filtered);
+            Assert.Equivalent(Array.Empty<string>(), progressMonitor.Messages, strict: true);
+        }
+
+        [Fact]
+        public void TestFiltersWithOnlyInclude()
+        {
+            (var testSubject, var progressMonitor, var files) = TestSetup();
+
+            Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
+                include:c/d
+                include:c/x/y
+                """);
+
+            var filtered = testSubject.Filter(files);
+
+            var expected = GetExpected(
+                [
+                    "/a/b/c/d/e/f.cs",
+                    "/a/b/c/d/e/g.cs",
+                    "/a/b/c/d/e/h.cs",
+                    "/a/b/c/x/y/i.cs"
+                ]);
+
+            AssertFileInfoEquivalence(expected, filtered);
+
+            var expectedRegexMessages = new[]
+            {
+                "Filtering in files matching '^c/d.*'. Original glob filter: 'include:c/d'",
+                "Filtering in files matching '^c/x/y.*'. Original glob filter: 'include:c/x/y'"
+            };
+            Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
+        }
+
+        [Fact]
+        public void TestFiltersWithOnlyExclude()
+        {
+            (var testSubject, var progressMonitor, var files) = TestSetup();
+
+            Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
+                exclude:c/d/e
+                """);
+
+            var filtered = testSubject.Filter(files);
+
+            var expected = GetExpected(
+                [
+                    "/a/b/c/x/y/i.cs",
+                    "/a/b/c/x/z/i.cs"
+                ]);
+
+            AssertFileInfoEquivalence(expected, filtered);
+
+            var expectedRegexMessages = new[]
+            {
+                "Filtering out files matching '^c/d/e.*'. Original glob filter: 'exclude:c/d/e'"
+            };
+            Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
+        }
+
+        [Fact]
+        public void TestFiltersWithIncludeExclude()
+        {
+            (var testSubject, var progressMonitor, var files) = TestSetup();
+
+            Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
+                include:c/x
+                exclude:c/x/z
+                """);
+
+            var filtered = testSubject.Filter(files);
+
+            var expected = GetExpected(
+                 [
+                     "/a/b/c/x/y/i.cs"
+                 ]);
+
+            AssertFileInfoEquivalence(expected, filtered);
+
+            var expectedRegexMessages = new[]
+            {
+                "Filtering in files matching '^c/x.*'. Original glob filter: 'include:c/x'",
+                "Filtering out files matching '^c/x/z.*'. Original glob filter: 'exclude:c/x/z'"
+            };
+            Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
+        }
+
+        [Fact]
+        public void TestFiltersWithIncludeExcludeExcludeFirst()
+        {
+            (var testSubject, var progressMonitor, var files) = TestSetup();
+
+            Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
+                exclude:c/x/z
+                include:c/x
+                """);
+
+            var filtered = testSubject.Filter(files);
+
+            var expected = GetExpected(
+                [
+                    "/a/b/c/x/y/i.cs"
+                ]);
+
+            AssertFileInfoEquivalence(expected, filtered);
+
+            var expectedRegexMessages = new[]
+            {
+                "Filtering in files matching '^c/x.*'. Original glob filter: 'include:c/x'",
+                "Filtering out files matching '^c/x/z.*'. Original glob filter: 'exclude:c/x/z'"
+            };
+            Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
+        }
+
+        [Fact]
+        public void TestFiltersWithIncludeExcludeComplexPatterns1()
+        {
+            (var testSubject, var progressMonitor, var files) = TestSetup();
+
+            Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
+                include:c/**/i.*
+                include:c/d/**/*.cs
+                exclude:**/z/i.cs
+                """);
+
+            var filtered = testSubject.Filter(files);
+
+            var expected = GetExpected(
+                [
+                    "/a/b/c/d/e/f.cs",
+                    "/a/b/c/d/e/g.cs",
+                    "/a/b/c/d/e/h.cs",
+                    "/a/b/c/x/y/i.cs"
+                ]);
+
+            AssertFileInfoEquivalence(expected, filtered);
+
+            var expectedRegexMessages = new[]
+            {
+                "Filtering in files matching '^c/.*/i\\.[^/]*.*'. Original glob filter: 'include:c/**/i.*'",
+                "Filtering in files matching '^c/d/.*/[^/]*\\.cs.*'. Original glob filter: 'include:c/d/**/*.cs'",
+                "Filtering out files matching '^.*/z/i\\.cs.*'. Original glob filter: 'exclude:**/z/i.cs'"
+            };
+            Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
+        }
+
+        [Fact]
+        public void TestFiltersWithIncludeExcludeComplexPatterns2()
+        {
+            (var testSubject, var progressMonitor, var files) = TestSetup();
+
+            Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
+                include:**/i.*
+                exclude:**/z/i.cs
+                """);
+
+            var filtered = testSubject.Filter(files);
+
+            var expected = GetExpected(
+                [
+                    "/a/b/c/x/y/i.cs"
+                ]);
+
+            AssertFileInfoEquivalence(expected, filtered);
+
+            var expectedRegexMessages = new[]
+            {
+                "Filtering in files matching '^.*/i\\.[^/]*.*'. Original glob filter: 'include:**/i.*'",
+                "Filtering out files matching '^.*/z/i\\.cs.*'. Original glob filter: 'exclude:**/z/i.cs'"
+            };
+            Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.Tests/Options.cs

@@ -135,23 +135,14 @@ namespace Semmle.Extraction.Tests
         public void StandaloneDefaults()
         {
             standaloneOptions = CSharp.Standalone.Options.Create(Array.Empty<string>());
-            Assert.Empty(standaloneOptions.Dependencies.DllDirs);
-            Assert.True(standaloneOptions.Dependencies.UseNuGet);
-            Assert.False(standaloneOptions.SkipExtraction);
-            Assert.Null(standaloneOptions.Dependencies.SolutionFile);
-            Assert.True(standaloneOptions.Dependencies.ScanNetFrameworkDlls);
             Assert.False(standaloneOptions.Errors);
         }
 
         [Fact]
         public void StandaloneOptions()
         {
-            standaloneOptions = CSharp.Standalone.Options.Create(new string[] { "--references:foo", "--silent", "--skip-nuget", "--skip-dotnet", "--exclude", "bar" });
-            Assert.Equal("foo", standaloneOptions.Dependencies.DllDirs[0]);
-            Assert.Equal("bar", standaloneOptions.Dependencies.Excludes[0]);
+            standaloneOptions = CSharp.Standalone.Options.Create(new string[] { "--silent" });
             Assert.Equal(Verbosity.Off, standaloneOptions.Verbosity);
-            Assert.False(standaloneOptions.Dependencies.UseNuGet);
-            Assert.False(standaloneOptions.Dependencies.ScanNetFrameworkDlls);
             Assert.False(standaloneOptions.Errors);
             Assert.False(standaloneOptions.Help);
         }
@@ -159,7 +150,7 @@ namespace Semmle.Extraction.Tests
         [Fact]
         public void InvalidOptions()
         {
-            standaloneOptions = CSharp.Standalone.Options.Create(new string[] { "--references:foo", "--silent", "--no-such-option" });
+            standaloneOptions = CSharp.Standalone.Options.Create(new string[] { "--silent", "--no-such-option" });
             Assert.True(standaloneOptions.Errors);
         }