Merge pull request #18029 from github/mbg/csharp/set-proxy-cert-file

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs

@@ -0,0 +1,87 @@
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Security.Cryptography.X509Certificates;
+using Semmle.Util;
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    public class DependabotProxy : IDisposable
+    {
+        private readonly string host;
+        private readonly string port;
+
+        /// <summary>
+        /// The full address of the Dependabot proxy, if available.
+        /// </summary>
+        internal string Address { get; }
+        /// <summary>
+        /// The path to the temporary file where the certificate is stored.
+        /// </summary>
+        internal string? CertificatePath { get; private set; }
+        /// <summary>
+        /// The certificate used for the Dependabot proxy.
+        /// </summary>
+        internal X509Certificate2? Certificate { get; private set; }
+
+        internal static DependabotProxy? GetDependabotProxy(ILogger logger, TemporaryDirectory tempWorkingDirectory)
+        {
+            // Setting HTTP(S)_PROXY and SSL_CERT_FILE have no effect on Windows or macOS,
+            // but we would still end up using the Dependabot proxy to check for feed reachability.
+            // This would result in us discovering that the feeds are reachable, but `dotnet` would
+            // fail to connect to them. To prevent this from happening, we do not initialise an
+            // instance of `DependabotProxy` on those platforms.
+            if (SystemBuildActions.Instance.IsWindows() || SystemBuildActions.Instance.IsMacOs()) return null;
+
+            // Obtain and store the address of the Dependabot proxy, if available.
+            var host = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ProxyHost);
+            var port = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ProxyPort);
+
+            if (string.IsNullOrWhiteSpace(host) || string.IsNullOrWhiteSpace(port))
+            {
+                logger.LogInfo("No Dependabot proxy credentials are configured.");
+                return null;
+            }
+
+            var result = new DependabotProxy(host, port);
+            logger.LogInfo($"Dependabot proxy configured at {result.Address}");
+
+            // Obtain and store the proxy's certificate, if available.
+            var cert = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ProxyCertificate);
+
+            if (!string.IsNullOrWhiteSpace(cert))
+            {
+                logger.LogInfo("No certificate configured for Dependabot proxy.");
+
+                var certDirPath = new DirectoryInfo(Path.Join(tempWorkingDirectory.DirInfo.FullName, ".dependabot-proxy"));
+                Directory.CreateDirectory(certDirPath.FullName);
+
+                result.CertificatePath = Path.Join(certDirPath.FullName, "proxy.crt");
+                var certFile = new FileInfo(result.CertificatePath);
+
+                using var writer = certFile.CreateText();
+                writer.Write(cert);
+                writer.Close();
+
+                logger.LogInfo($"Stored Dependabot proxy certificate at {result.CertificatePath}");
+
+                result.Certificate = X509Certificate2.CreateFromPem(cert);
+            }
+
+            return result;
+        }
+
+        private DependabotProxy(string host, string port)
+        {
+            this.host = host;
+            this.port = port;
+            this.Address = $"http://{this.host}:{this.port}";
+        }
+
+        public void Dispose()
+        {
+            this.Certificate?.Dispose();
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -27,6 +27,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         private readonly ILogger logger;
         private readonly IDiagnosticsWriter diagnosticsWriter;
         private readonly NugetPackageRestorer nugetPackageRestorer;
+        private readonly DependabotProxy? dependabotProxy;
         private readonly IDotNet dotnet;
         private readonly FileContent fileContent;
         private readonly FileProvider fileProvider;
@@ -106,9 +107,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 return BuildScript.Success;
             }).Run(SystemBuildActions.Instance, startCallback, exitCallback);
 
+            dependabotProxy = DependabotProxy.GetDependabotProxy(logger, tempWorkingDirectory);
+
             try
             {
-                this.dotnet = DotNet.Make(logger, dotnetPath, tempWorkingDirectory);
+                this.dotnet = DotNet.Make(logger, dotnetPath, tempWorkingDirectory, dependabotProxy);
                 runtimeLazy = new Lazy<Runtime>(() => new Runtime(dotnet));
             }
             catch
@@ -117,7 +120,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 throw;
             }
 
-            nugetPackageRestorer = new NugetPackageRestorer(fileProvider, fileContent, dotnet, diagnosticsWriter, logger, this);
+            nugetPackageRestorer = new NugetPackageRestorer(fileProvider, fileContent, dotnet, dependabotProxy, diagnosticsWriter, logger, this);
 
             var dllLocations = fileProvider.Dlls.Select(x => new AssemblyLookupLocation(x)).ToHashSet();
             dllLocations.UnionWith(nugetPackageRestorer.Restore());
@@ -542,6 +545,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         public void Dispose()
         {
             nugetPackageRestorer?.Dispose();
+            dependabotProxy?.Dispose();
             if (cleanupTempWorkingDirectory)
             {
                 tempWorkingDirectory?.Dispose();

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -27,11 +27,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             Info();
         }
 
-        private DotNet(ILogger logger, string? dotNetPath, TemporaryDirectory tempWorkingDirectory) : this(new DotNetCliInvoker(logger, Path.Combine(dotNetPath ?? string.Empty, "dotnet")), logger, tempWorkingDirectory) { }
+        private DotNet(ILogger logger, string? dotNetPath, TemporaryDirectory tempWorkingDirectory, DependabotProxy? dependabotProxy) : this(new DotNetCliInvoker(logger, Path.Combine(dotNetPath ?? string.Empty, "dotnet"), dependabotProxy), logger, tempWorkingDirectory) { }
 
         internal static IDotNet Make(IDotNetCliInvoker dotnetCliInvoker, ILogger logger) => new DotNet(dotnetCliInvoker, logger);
 
-        public static IDotNet Make(ILogger logger, string? dotNetPath, TemporaryDirectory tempWorkingDirectory) => new DotNet(logger, dotNetPath, tempWorkingDirectory);
+        public static IDotNet Make(ILogger logger, string? dotNetPath, TemporaryDirectory tempWorkingDirectory, DependabotProxy? dependabotProxy) => new DotNet(logger, dotNetPath, tempWorkingDirectory, dependabotProxy);
 
         private void Info()
         {

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs

@@ -12,12 +12,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     internal sealed class DotNetCliInvoker : IDotNetCliInvoker
     {
         private readonly ILogger logger;
+        private readonly DependabotProxy? proxy;
 
         public string Exec { get; }
 
-        public DotNetCliInvoker(ILogger logger, string exec)
+        public DotNetCliInvoker(ILogger logger, string exec, DependabotProxy? dependabotProxy)
         {
             this.logger = logger;
+            this.proxy = dependabotProxy;
             this.Exec = exec;
             logger.LogInfo($"Using .NET CLI executable: '{Exec}'");
         }
@@ -38,6 +40,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             startInfo.EnvironmentVariables["DOTNET_CLI_UI_LANGUAGE"] = "en";
             startInfo.EnvironmentVariables["MSBUILDDISABLENODEREUSE"] = "1";
             startInfo.EnvironmentVariables["DOTNET_SKIP_FIRST_TIME_EXPERIENCE"] = "true";
+
+            // Configure the proxy settings, if applicable.
+            if (this.proxy != null)
+            {
+                logger.LogInfo($"Setting up Dependabot proxy at {this.proxy.Address}");
+
+                startInfo.EnvironmentVariables.Add("HTTP_PROXY", this.proxy.Address);
+                startInfo.EnvironmentVariables.Add("HTTPS_PROXY", this.proxy.Address);
+                startInfo.EnvironmentVariables.Add("SSL_CERT_FILE", this.proxy.CertificatePath);
+            }
+
             return startInfo;
         }
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

@@ -74,5 +74,20 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         /// Specifies the location of the diagnostic directory.
         /// </summary>
         public const string DiagnosticDir = "CODEQL_EXTRACTOR_CSHARP_DIAGNOSTIC_DIR";
+
+        /// <summary>
+        /// Specifies the hostname of the Dependabot proxy.
+        /// </summary>
+        public const string ProxyHost = "CODEQL_PROXY_HOST";
+
+        /// <summary>
+        /// Specifies the hostname of the Dependabot proxy.
+        /// </summary>
+        public const string ProxyPort = "CODEQL_PROXY_PORT";
+
+        /// <summary>
+        /// Contains the certificate used by the Dependabot proxy.
+        /// </summary>
+        public const string ProxyCertificate = "CODEQL_PROXY_CA_CERTIFICATE";
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -3,7 +3,9 @@ using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Net;
 using System.Net.Http;
+using System.Security.Cryptography.X509Certificates;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading;
@@ -20,6 +22,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         private readonly FileProvider fileProvider;
         private readonly FileContent fileContent;
         private readonly IDotNet dotnet;
+        private readonly DependabotProxy? dependabotProxy;
         private readonly IDiagnosticsWriter diagnosticsWriter;
         private readonly TemporaryDirectory legacyPackageDirectory;
         private readonly TemporaryDirectory missingPackageDirectory;
@@ -32,6 +35,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             FileProvider fileProvider,
             FileContent fileContent,
             IDotNet dotnet,
+            DependabotProxy? dependabotProxy,
             IDiagnosticsWriter diagnosticsWriter,
             ILogger logger,
             ICompilationInfoContainer compilationInfoContainer)
@@ -39,6 +43,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             this.fileProvider = fileProvider;
             this.fileContent = fileContent;
             this.dotnet = dotnet;
+            this.dependabotProxy = dependabotProxy;
             this.diagnosticsWriter = diagnosticsWriter;
             this.logger = logger;
             this.compilationInfoContainer = compilationInfoContainer;
@@ -588,7 +593,25 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         private bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount, bool allowExceptions = true)
         {
             logger.LogInfo($"Checking if Nuget feed '{feed}' is reachable...");
-            using HttpClient client = new();
+
+            // Configure the HttpClient to be aware of the Dependabot Proxy, if used.
+            HttpClientHandler httpClientHandler = new();
+            if (this.dependabotProxy != null)
+            {
+                httpClientHandler.Proxy = new WebProxy(this.dependabotProxy.Address);
+
+                if (this.dependabotProxy.Certificate != null)
+                {
+                    httpClientHandler.ServerCertificateCustomValidationCallback = (message, cert, chain, _) =>
+                    {
+                        chain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust;
+                        chain.ChainPolicy.CustomTrustStore.Add(this.dependabotProxy.Certificate);
+                        return chain.Build(cert);
+                    };
+                }
+            }
+
+            using HttpClient client = new(httpClientHandler);
 
             for (var i = 0; i < tryCount; i++)
             {