C#: Add paths/paths-ignore support in standalone

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -439,6 +439,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 files = files.Where(f => !f.FullName.StartsWith(options.DotNetPath, StringComparison.OrdinalIgnoreCase));
             }
 
+            files = new FilePathFilter(sourceDir, progressMonitor).Filter(files);
             return files;
         }
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs

@@ -0,0 +1,81 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Security.Cryptography.X509Certificates;
+using System.Text.RegularExpressions;
+using Semmle.Util;
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    public class FilePathFilter
+    {
+        private readonly string rootFolder;
+        private readonly IProgressMonitor progressMonitor;
+
+        public FilePathFilter(DirectoryInfo sourceDir, IProgressMonitor progressMonitor)
+        {
+            rootFolder = FileUtils.ConvertToUnix(sourceDir.FullName.ToLowerInvariant());
+            this.progressMonitor = progressMonitor;
+        }
+
+        private class FileInclusion(string path, bool include)
+        {
+            public string Path { get; } = path;
+
+            public bool Include { get; set; } = include;
+        }
+
+        private record class PathFilter(Regex Regex, bool Include);
+
+        public IEnumerable<FileInfo> Filter(IEnumerable<FileInfo> files)
+        {
+            var filters = (Environment.GetEnvironmentVariable("LGTM_INDEX_FILTERS") ?? string.Empty).Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
+            if (filters.Length == 0)
+            {
+                return files;
+            }
+
+            var pathFilters = new List<PathFilter>();
+
+            foreach (var filter in filters)
+            {
+                bool include;
+                string filterText;
+                if (filter.StartsWith("include:"))
+                {
+                    include = true;
+                    filterText = filter.Substring("include:".Length);
+                }
+                else if (filter.StartsWith("exclude:"))
+                {
+                    include = false;
+                    filterText = filter.Substring("exclude:".Length);
+                }
+                else
+                {
+                    progressMonitor.Log(Severity.Info, $"Invalid filter: {filter}");
+                    continue;
+                }
+
+                pathFilters.Add(new PathFilter(new Regex(new FilePattern(filterText).RegexPattern, RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline), include));
+            }
+
+            var fileIndex = files.ToDictionary(f => f, f => new FileInclusion(FileUtils.ConvertToUnix(f.FullName.ToLowerInvariant()).Replace(rootFolder, string.Empty).TrimStart('/'), pathFilters.All(f => !f.Include)));
+
+            foreach (var pathFilter in pathFilters.OrderBy(pf => pf.Include ? 0 : 1))
+            {
+                foreach (var file in fileIndex)
+                {
+                    if (pathFilter.Regex.IsMatch(file.Value.Path))
+                    {
+                        fileIndex[file.Key].Include = pathFilter.Include;
+                    }
+                }
+            }
+
+            return fileIndex.Where(f => f.Value.Include).Select(f => f.Key);
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IProgressMonitor.cs

@@ -0,0 +1,9 @@
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    public interface IProgressMonitor
+    {
+        void Log(Severity severity, string message);
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs

@@ -3,7 +3,7 @@ using Semmle.Util.Logging;
 
 namespace Semmle.Extraction.CSharp.DependencyFetching
 {
-    internal class ProgressMonitor
+    internal class ProgressMonitor : IProgressMonitor
     {
         private readonly ILogger logger;
 

csharp/extractor/Semmle.Extraction.Tests/FilePathFilter.cs

@@ -0,0 +1,238 @@
+using Xunit;
+using Semmle.Extraction.CSharp.DependencyFetching;
+using Semmle.Util.Logging;
+using System.Linq;
+using System.Collections.Generic;
+using System.IO;
+using System;
+
+namespace Semmle.Extraction.Tests
+{
+    public class FilePathFilterTest
+    {
+        private class ProgressMonitorStub : IProgressMonitor
+        {
+            public void Log(Severity severity, string message) { }
+        }
+
+        private static (FilePathFilter TestSubject, IEnumerable<FileInfo> Files) TestSetup(string root, IEnumerable<string> paths)
+        {
+            root = GetPlatformSpecifixPath(root);
+            paths = GetPlatformSpecifixPaths(paths);
+
+            var filePathFilter = new FilePathFilter(new DirectoryInfo(root), new ProgressMonitorStub());
+            return (filePathFilter, paths.Select(p => new FileInfo(p)));
+        }
+
+        private static string GetPlatformSpecifixPath(string file)
+        {
+            return file.Replace('/', Path.DirectorySeparatorChar);
+        }
+
+        private static IEnumerable<string> GetPlatformSpecifixPaths(IEnumerable<string> files)
+        {
+            return files.Select(GetPlatformSpecifixPath);
+        }
+
+        private static IEnumerable<FileInfo> GetExpected(IEnumerable<string> files)
+        {
+            files = GetPlatformSpecifixPaths(files);
+            return files.Select(f => new FileInfo(f));
+        }
+
+        private static void AssertEquivalence(IEnumerable<FileInfo>? expected, IEnumerable<FileInfo>? actual)
+        {
+            Assert.Equivalent(expected?.Select(f => f.FullName), actual?.Select(f => f.FullName), strict: true);
+        }
+
+        [Fact]
+        public void TestNoFilter()
+        {
+            (var testSubject, var files) = TestSetup(
+                "/a/b",
+                [
+                    "/a/b/c/d/e/f.cs",
+                    "/a/b/c/d/e/g.cs",
+                    "/a/b/c/d/e/h.cs",
+                    "/a/b/c/x/y/i.cs",
+                    "/a/b/c/x/z/i.cs"
+                ]);
+
+            Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", null);
+
+            var filtered = testSubject.Filter(files);
+
+            AssertEquivalence(files, filtered);
+        }
+
+        [Fact]
+        public void TestFiltersWithOnlyInclude()
+        {
+            (var testSubject, var files) = TestSetup(
+                "/a/b",
+                [
+                    "/a/b/c/d/e/f.cs",
+                    "/a/b/c/d/e/g.cs",
+                    "/a/b/c/d/e/h.cs",
+                    "/a/b/c/x/y/i.cs",
+                    "/a/b/c/x/z/i.cs"
+                ]);
+
+            Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
+                include:c/d
+                include:c/x/y
+                """);
+
+            var filtered = testSubject.Filter(files);
+
+            var expected = GetExpected(
+                [
+                    "/a/b/c/d/e/f.cs",
+                    "/a/b/c/d/e/g.cs",
+                    "/a/b/c/d/e/h.cs",
+                    "/a/b/c/x/y/i.cs"
+                ]);
+
+            AssertEquivalence(expected, filtered);
+        }
+
+        [Fact]
+        public void TestFiltersWithOnlyExclude()
+        {
+            (var testSubject, var files) = TestSetup("/a/b",
+            [
+                "/a/b/c/d/e/f.cs",
+                "/a/b/c/d/e/g.cs",
+                "/a/b/c/d/e/h.cs",
+                "/a/b/c/x/y/i.cs",
+                "/a/b/c/x/z/i.cs"
+            ]);
+
+            Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
+                exclude:c/d/e
+                """);
+
+            var filtered = testSubject.Filter(files);
+
+            var expected = GetExpected(
+                [
+                    "/a/b/c/x/y/i.cs",
+                    "/a/b/c/x/z/i.cs"
+                ]);
+
+            AssertEquivalence(expected, filtered);
+        }
+
+        [Fact]
+        public void TestFiltersWithIncludeExclude()
+        {
+            (var testSubject, var files) = TestSetup("/a/b",
+            [
+                "/a/b/c/d/e/f.cs",
+                "/a/b/c/d/e/g.cs",
+                "/a/b/c/d/e/h.cs",
+                "/a/b/c/x/y/i.cs",
+                "/a/b/c/x/z/i.cs"
+            ]);
+
+            Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
+                include:c/x
+                exclude:c/x/z
+                """);
+
+            var filtered = testSubject.Filter(files);
+
+            var expected = GetExpected(
+                 [
+                     "/a/b/c/x/y/i.cs"
+                 ]);
+
+            AssertEquivalence(expected, filtered);
+        }
+
+        [Fact]
+        public void TestFiltersWithIncludeExcludeExcludeFirst()
+        {
+            (var testSubject, var files) = TestSetup("/a/b",
+            [
+                "/a/b/c/d/e/f.cs",
+                "/a/b/c/d/e/g.cs",
+                "/a/b/c/d/e/h.cs",
+                "/a/b/c/x/y/i.cs",
+                "/a/b/c/x/z/i.cs"
+            ]);
+
+            Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
+                exclude:c/x/z
+                include:c/x
+                """);
+
+            var filtered = testSubject.Filter(files);
+
+            var expected = GetExpected(
+                [
+                    "/a/b/c/x/y/i.cs"
+                ]);
+
+            AssertEquivalence(expected, filtered);
+        }
+
+        [Fact]
+        public void TestFiltersWithIncludeExcludeComplexPatterns1()
+        {
+            (var testSubject, var files) = TestSetup("/a/b",
+            [
+                "/a/b/c/d/e/f.cs",
+                "/a/b/c/d/e/g.cs",
+                "/a/b/c/d/e/h.cs",
+                "/a/b/c/x/y/i.cs",
+                "/a/b/c/x/z/i.cs"
+            ]);
+
+            Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
+                include:c/**/i.*
+                include:c/d/**/*.cs
+                exclude:**/z/i.cs
+                """);
+
+            var filtered = testSubject.Filter(files);
+
+            var expected = GetExpected(
+                [
+                    "/a/b/c/d/e/f.cs",
+                    "/a/b/c/d/e/g.cs",
+                    "/a/b/c/d/e/h.cs",
+                    "/a/b/c/x/y/i.cs"
+                ]);
+
+            AssertEquivalence(expected, filtered);
+        }
+
+        [Fact]
+        public void TestFiltersWithIncludeExcludeComplexPatterns2()
+        {
+            (var testSubject, var files) = TestSetup("/a/b",
+            [
+                "/a/b/c/d/e/f.cs",
+                "/a/b/c/d/e/g.cs",
+                "/a/b/c/d/e/h.cs",
+                "/a/b/c/x/y/i.cs",
+                "/a/b/c/x/z/i.cs"
+            ]);
+
+            Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
+                include:**/i.*
+                exclude:**/z/i.cs
+                """);
+
+            var filtered = testSubject.Filter(files);
+
+            var expected = GetExpected(
+                [
+                    "/a/b/c/x/y/i.cs"
+                ]);
+
+            AssertEquivalence(expected, filtered);
+        }
+    }
+}