Use proxy for feed reachability check

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetConfig.cs

@@ -0,0 +1,74 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using Semmle.Util;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    public class NugetConfig
+    {
+        internal class NugetFeed : IComparable<NugetFeed>
+        {
+            internal string Value { get; }
+            internal bool DisableTlsCertificateValidation { get; set; }
+
+            internal NugetFeed(string value)
+            {
+                this.Value = value;
+                this.DisableTlsCertificateValidation = true;
+            }
+
+            public override string ToString()
+            {
+                return this.Value;
+            }
+
+            public int CompareTo(NugetFeed? other)
+            {
+                return this.Value.CompareTo(other?.Value);
+            }
+        }
+
+        internal IEnumerable<NugetFeed> Feeds { get; }
+
+        public NugetConfig()
+        {
+            this.Feeds = new List<NugetFeed>();
+        }
+
+        /// <summary>
+        /// Writes this configuration to a file located at <paramref name="nugetConfigPath"/>.
+        /// </summary>
+        /// <param name="nugetConfigPath">The path of the file to which the configuration should be written to.</param>
+        public void Write(string nugetConfigPath)
+        {
+            var config = "";
+
+            var proxyHost = Environment.GetEnvironmentVariable("CODEQL_PROXY_HOST");
+            var proxyPort = Environment.GetEnvironmentVariable("CODEQL_PROXY_PORT");
+            if (!string.IsNullOrWhiteSpace(proxyHost) && !string.IsNullOrWhiteSpace(proxyPort))
+            {
+                var proxyAddress = $"http://{proxyHost}:{proxyPort}";
+                config = $"""<add key="http_proxy" value="{proxyAddress}" />""";
+            }
+
+            var sb = new StringBuilder();
+            this.Feeds.ForEach((feed, index) => sb.AppendLine($"<add key=\"feed{index}\" value=\"{feed.Value}\" disableTLSCertificateValidation=\"{feed.DisableTlsCertificateValidation}\" />"));
+
+            File.WriteAllText(nugetConfigPath,
+                $"""
+                <?xml version="1.0" encoding="utf-8"?>
+                <configuration>
+                    <config>
+                        {config}
+                    </config>
+                    <packageSources>
+                        <clear />
+                        {sb}
+                    </packageSources>
+                </configuration>
+                """);
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -3,6 +3,7 @@ using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Net;
 using System.Net.Http;
 using System.Security.Cryptography;
 using System.Text;
@@ -93,12 +94,25 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
         public HashSet<AssemblyLookupLocation> Restore()
         {
+            try
+            {
+                var userConfigPath = Path.Join(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".nuget", "NuGet");
+                Directory.CreateDirectory(userConfigPath);
+
+                var userConfig = new NugetConfig();
+                userConfig.Write(Path.Join(userConfigPath, "NuGet.Config"));
+            }
+            catch (Exception ex)
+            {
+                logger.LogError($"Failed to write user nuget.config: {ex}");
+            }
+
             var assemblyLookupLocations = new HashSet<AssemblyLookupLocation>();
             var checkNugetFeedResponsiveness = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.CheckNugetFeedResponsiveness);
             logger.LogInfo($"Checking NuGet feed responsiveness: {checkNugetFeedResponsiveness}");
             compilationInfoContainer.CompilationInfos.Add(("NuGet feed responsiveness checked", checkNugetFeedResponsiveness ? "1" : "0"));
 
-            HashSet<string>? explicitFeeds = null;
+            HashSet<NugetConfig.NugetFeed>? explicitFeeds = null;
 
             try
             {
@@ -175,12 +189,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return assemblyLookupLocations;
         }
 
-        private List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNugetConfigs)
+        private List<NugetConfig.NugetFeed> GetReachableFallbackNugetFeeds(HashSet<NugetConfig.NugetFeed>? feedsFromNugetConfigs)
         {
-            var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet();
+            var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).Select(feed => new NugetConfig.NugetFeed(feed)).ToHashSet();
             if (fallbackFeeds.Count == 0)
             {
-                fallbackFeeds.Add(PublicNugetOrgFeed);
+                fallbackFeeds.Add(new NugetConfig.NugetFeed(PublicNugetOrgFeed));
                 logger.LogInfo($"No fallback Nuget feeds specified. Adding default feed: {PublicNugetOrgFeed}");
 
                 var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback);
@@ -293,7 +307,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             compilationInfoContainer.CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
         }
 
-        private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds(HashSet<string>? feedsFromNugetConfigs)
+        private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds(HashSet<NugetConfig.NugetFeed>? feedsFromNugetConfigs)
         {
             var reachableFallbackFeeds = GetReachableFallbackNugetFeeds(feedsFromNugetConfigs);
             if (reachableFallbackFeeds.Count > 0)
@@ -305,7 +319,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return null;
         }
 
-        private AssemblyLookupLocation? DownloadMissingPackages(IEnumerable<string>? fallbackNugetFeeds = null)
+        private AssemblyLookupLocation? DownloadMissingPackages(IEnumerable<NugetConfig.NugetFeed>? fallbackNugetFeeds = null)
         {
             var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(PackageDirectory.DirInfo);
             var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
@@ -367,23 +381,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return missingPackageDirectory.DirInfo.FullName;
         }
 
-        private string? CreateFallbackNugetConfig(IEnumerable<string> fallbackNugetFeeds, string folderPath)
+        private string? CreateFallbackNugetConfig(IEnumerable<NugetConfig.NugetFeed> fallbackNugetFeeds, string folderPath)
         {
-            var sb = new StringBuilder();
-            fallbackNugetFeeds.ForEach((feed, index) => sb.AppendLine($"<add key=\"feed{index}\" value=\"{feed}\" />"));
+            NugetConfig config = new NugetConfig();
+            fallbackNugetFeeds.ForEach((feed, index) => config.Feeds.Append(feed));
 
             var nugetConfigPath = Path.Combine(folderPath, "nuget.config");
             logger.LogInfo($"Creating fallback nuget.config file {nugetConfigPath}.");
-            File.WriteAllText(nugetConfigPath,
-                $"""
-                <?xml version="1.0" encoding="utf-8"?>
-                <configuration>
-                    <packageSources>
-                        <clear />
-                {sb}
-                    </packageSources>
-                </configuration>
-                """);
+            config.Write(nugetConfigPath);
 
             return nugetConfigPath;
         }
@@ -586,10 +591,34 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             }
         }
 
-        private bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount, bool allowExceptions = true)
+        private bool IsFeedReachable(NugetConfig.NugetFeed feed, int timeoutMilliSeconds, int tryCount, bool allowExceptions = true)
         {
             logger.LogInfo($"Checking if Nuget feed '{feed}' is reachable...");
-            using HttpClient client = new();
+
+            // Configure the handler for this check. If `DisableTlsCertificateValidation` is `true` for this feed,
+            // we disable certificate validation.
+            var handler = new HttpClientHandler();
+
+            var proxyHost = Environment.GetEnvironmentVariable("CODEQL_PROXY_HOST");
+            var proxyPort = Environment.GetEnvironmentVariable("CODEQL_PROXY_PORT");
+            if (!string.IsNullOrWhiteSpace(proxyHost) && !string.IsNullOrWhiteSpace(proxyPort))
+            {
+                var proxyAddress = new Uri($"http://{proxyHost}:{proxyPort}");
+                handler.Proxy = new WebProxy(proxyAddress);
+                handler.Proxy.Credentials = new NetworkCredential(Environment.GetEnvironmentVariable("CODEQL_PROXY_USER"), Environment.GetEnvironmentVariable("CODEQL_PROXY_PASSWORD"));
+
+                logger.LogInfo($"Using proxy at {proxyAddress}...");
+            }
+
+            if (feed.DisableTlsCertificateValidation)
+            {
+                logger.LogInfo($"Disabling TLS certificate validation for '{feed}'...");
+                handler.ClientCertificateOptions = ClientCertificateOption.Manual;
+                handler.ServerCertificateCustomValidationCallback =
+                    (httpRequestMessage, cert, certChain, policyErrors) => { return true; };
+            }
+
+            using HttpClient client = new(handler);
 
             for (var i = 0; i < tryCount; i++)
             {
@@ -597,7 +626,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 cts.CancelAfter(timeoutMilliSeconds);
                 try
                 {
-                    ExecuteGetRequest(feed, client, cts.Token).GetAwaiter().GetResult();
+                    ExecuteGetRequest(feed.Value, client, cts.Token).GetAwaiter().GetResult();
                     logger.LogInfo($"Querying Nuget feed '{feed}' succeeded.");
                     return true;
                 }
@@ -615,6 +644,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                     // We're only interested in timeouts.
                     var start = allowExceptions ? "Considering" : "Not considering";
                     logger.LogInfo($"Querying Nuget feed '{feed}' failed in a timely manner. {start} the feed for use. The reason for the failure: {exc.Message}");
+                    logger.LogError($"Exception: {exc}");
+
+                    if (exc.InnerException != null)
+                    {
+                        logger.LogError($"Inner exception: {exc.InnerException}");
+                    }
+
                     return allowExceptions;
                 }
             }
@@ -642,7 +678,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return (timeoutMilliSeconds, tryCount);
         }
 
-        private bool CheckFeeds(out HashSet<string> explicitFeeds)
+        private bool CheckFeeds(out HashSet<NugetConfig.NugetFeed> explicitFeeds)
         {
             logger.LogInfo("Checking Nuget feeds...");
             (explicitFeeds, var allFeeds) = GetAllFeeds();
@@ -657,7 +693,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
             var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false);
 
-            var allFeedsReachable = explicitFeeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed, initialTimeout, tryCount));
+            var allFeedsReachable = explicitFeeds.All(feed => excludedFeeds.Contains(feed.Value) || IsFeedReachable(feed, initialTimeout, tryCount));
             if (!allFeedsReachable)
             {
                 logger.LogWarning("Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.");
@@ -711,11 +747,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             }
         }
 
-        private (HashSet<string> explicitFeeds, HashSet<string> allFeeds) GetAllFeeds()
+        private (HashSet<NugetConfig.NugetFeed> explicitFeeds, HashSet<NugetConfig.NugetFeed> allFeeds) GetAllFeeds()
         {
             var nugetConfigs = fileProvider.NugetConfigs;
             var explicitFeeds = nugetConfigs
                 .SelectMany(config => GetFeeds(() => dotnet.GetNugetFeeds(config)))
+                .Select(feed => new NugetConfig.NugetFeed(feed))
                 .ToHashSet();
 
             if (explicitFeeds.Count > 0)
@@ -744,6 +781,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 })
                 .Where(folder => folder != null)
                 .SelectMany(folder => GetFeeds(() => dotnet.GetNugetFeedsFromFolder(folder!)))
+                .Select(feed => new NugetConfig.NugetFeed(feed))
                 .ToHashSet();
 
             logger.LogInfo($"Found {allFeeds.Count} Nuget feeds (with inherited ones) in nuget.config files: {string.Join(", ", allFeeds.OrderBy(f => f))}");