hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-25 08:45:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #17253 from tamasvajk/impr/add-retry-logic-to-file-download

Browse Source 
C#: Add retry logic to file (nuget.exe, dotnet-install.sh) downloads
This commit is contained in:
Tamás Vajk
2024-08-21 11:39:14 +02:00
committed by GitHub
parent 68dfca49f5 73674e72e6
commit b6255571d8
8 changed files with 53 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs
View File

@@ -7,6 +7,7 @@ using System.Xml;
using Microsoft.Build.Construction;
using Semmle.Util;
using Semmle.Autobuild.Shared;
using Semmle.Util.Logging;
namespace Semmle.Autobuild.CSharp.Tests
{
@@ -203,7 +204,7 @@ namespace Semmle.Autobuild.CSharp.Tests
throw new ArgumentException($"Missing CreateDirectory, {path}");
}
public void DownloadFile(string address, string fileName)
public void DownloadFile(string address, string fileName, ILogger logger)
{
if (!DownloadFiles.Contains((address, fileName)))
throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");

3
csharp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs
View File

@@ -7,6 +7,7 @@ using System.Linq;
using Microsoft.Build.Construction;
using System.Xml;
using System.IO;
using Semmle.Util.Logging;
namespace Semmle.Autobuild.Cpp.Tests
{
@@ -189,7 +190,7 @@ namespace Semmle.Autobuild.Cpp.Tests
throw new ArgumentException($"Missing CreateDirectory, {path}");
}
public void DownloadFile(string address, string fileName)
public void DownloadFile(string address, string fileName, ILogger logger)
{
if (!DownloadFiles.Contains((address, fileName)))
throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");

3
csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs
View File

@@ -157,7 +157,8 @@ namespace Semmle.Autobuild.Shared
BuildScript.DownloadFile(
FileUtils.NugetExeUrl,
path,
e => builder.Logger.LogWarning($"Failed to download 'nuget.exe': {e.Message}"))
e => builder.Logger.LogWarning($"Failed to download 'nuget.exe': {e.Message}"),
builder.Logger)
&
BuildScript.Create(_ =>
{

3
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
View File

@@ -248,7 +248,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var downloadDotNetInstallSh = BuildScript.DownloadFile(
"https://dot.net/v1/dotnet-install.sh",
dotnetInstallPath,
e => logger.LogWarning($"Failed to download 'dotnet-install.sh': {e.Message}"));
e => logger.LogWarning($"Failed to download 'dotnet-install.sh': {e.Message}"),
logger);
var chmod = new CommandBuilder(actions).
RunCommand("chmod").

2
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs
View File

@@ -145,7 +145,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
Directory.CreateDirectory(directory);
logger.LogInfo("Attempting to download nuget.exe");
FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget);
FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget, logger);
logger.LogInfo($"Downloaded nuget.exe to {nuget}");
return nuget;
}

8
csharp/extractor/Semmle.Util/BuildActions.cs
View File

@@ -6,7 +6,7 @@ using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Xml;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Util
{
@@ -165,7 +165,7 @@ namespace Semmle.Util
/// <summary>
/// Downloads the resource with the specified URI to a local file.
/// </summary>
void DownloadFile(string address, string fileName);
void DownloadFile(string address, string fileName, ILogger logger);
/// <summary>
/// Creates an <see cref="IDiagnosticsWriter" /> for the given <paramref name="filename" />.
@@ -280,8 +280,8 @@ namespace Semmle.Util
public string EnvironmentExpandEnvironmentVariables(string s) => Environment.ExpandEnvironmentVariables(s);
public void DownloadFile(string address, string fileName) =>
FileUtils.DownloadFile(address, fileName);
public void DownloadFile(string address, string fileName, ILogger logger) =>
FileUtils.DownloadFile(address, fileName, logger);
public IDiagnosticsWriter CreateDiagnosticsWriter(string filename) => new DiagnosticsStream(filename);

5
csharp/extractor/Semmle.Util/BuildScript.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using Semmle.Util.Logging;
namespace Semmle.Util
{
@@ -275,14 +276,14 @@ namespace Semmle.Util
/// <summary>
/// Creates a build script that downloads the specified file.
/// </summary>
public static BuildScript DownloadFile(string address, string fileName, Action<Exception> exceptionCallback) =>
public static BuildScript DownloadFile(string address, string fileName, Action<Exception> exceptionCallback, ILogger logger) =>
Create(actions =>
{
if (actions.GetDirectoryName(fileName) is string dir && !string.IsNullOrWhiteSpace(dir))
actions.CreateDirectory(dir);
try
{
actions.DownloadFile(address, fileName);
actions.DownloadFile(address, fileName, logger);
return 0;
}
catch (Exception e)

43
csharp/extractor/Semmle.Util/FileUtils.cs
View File

@@ -4,6 +4,7 @@ using System.Linq;
using System.Net.Http;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Semmle.Util.Logging;
@@ -99,19 +100,49 @@ namespace Semmle.Util
return hex.ToString();
}
private static async Task DownloadFileAsync(string address, string filename)
private static async Task DownloadFileAsync(string address, string filename, HttpClient httpClient, CancellationToken token)
{
using var httpClient = new HttpClient();
using var contentStream = await httpClient.GetStreamAsync(address);
using var contentStream = await httpClient.GetStreamAsync(address, token);
using var stream = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None, 4096, true);
await contentStream.CopyToAsync(stream);
await contentStream.CopyToAsync(stream, token);
}
private static void DownloadFileWithRetry(string address, string fileName, int tryCount, int timeoutMilliSeconds, ILogger logger)
{
logger.LogDebug($"Downloading {address} to {fileName}.");
using HttpClient client = new();
for (var i = 0; i < tryCount; i++)
{
logger.LogDebug($"Attempt {i + 1} of {tryCount}. Timeout: {timeoutMilliSeconds} ms.");
using var cts = new CancellationTokenSource();
cts.CancelAfter(timeoutMilliSeconds);
try
{
DownloadFileAsync(address, fileName, client, cts.Token).GetAwaiter().GetResult();
logger.LogDebug($"Downloaded {address} to {fileName}.");
return;
}
catch (Exception exc)
{
logger.LogDebug($"Failed to download {address} to {fileName}. Exception: {exc.Message}");
timeoutMilliSeconds *= 2;
if (i == tryCount - 1)
{
logger.LogDebug($"Failed to download {address} to {fileName} after {tryCount} attempts.");
// Rethrowing the last exception
throw;
}
}
}
}
/// <summary>
/// Downloads the file at <paramref name="address"/> to <paramref name="fileName"/>.
/// </summary>
public static void DownloadFile(string address, string fileName) =>
DownloadFileAsync(address, fileName).GetAwaiter().GetResult();
public static void DownloadFile(string address, string fileName, ILogger logger) =>
DownloadFileWithRetry(address, fileName, tryCount: 3, timeoutMilliSeconds: 10000, logger);
public static string ConvertPathToSafeRelativePath(string path)
{