hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-05-05 13:45:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

C#: Add retry logic to file (nuget.exe, dotnet-install.sh) downloads

Browse Source
This commit is contained in:
Tamas Vajk
2024-08-19 12:19:51 +02:00
parent a25d9c7397
commit 8b6c293b5c
8 changed files with 53 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
csharp/extractor/Semmle.Util/BuildActions.cs
View File

@@ -6,7 +6,7 @@ using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Xml;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Util
{
@@ -165,7 +165,7 @@ namespace Semmle.Util
/// <summary>
/// Downloads the resource with the specified URI to a local file.
/// </summary>
void DownloadFile(string address, string fileName);
void DownloadFile(string address, string fileName, ILogger logger);
/// <summary>
/// Creates an <see cref="IDiagnosticsWriter" /> for the given <paramref name="filename" />.
@@ -280,8 +280,8 @@ namespace Semmle.Util
public string EnvironmentExpandEnvironmentVariables(string s) => Environment.ExpandEnvironmentVariables(s);
public void DownloadFile(string address, string fileName) =>
FileUtils.DownloadFile(address, fileName);
public void DownloadFile(string address, string fileName, ILogger logger) =>
FileUtils.DownloadFile(address, fileName, logger);
public IDiagnosticsWriter CreateDiagnosticsWriter(string filename) => new DiagnosticsStream(filename);

5
csharp/extractor/Semmle.Util/BuildScript.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using Semmle.Util.Logging;
namespace Semmle.Util
{
@@ -275,14 +276,14 @@ namespace Semmle.Util
/// <summary>
/// Creates a build script that downloads the specified file.
/// </summary>
public static BuildScript DownloadFile(string address, string fileName, Action<Exception> exceptionCallback) =>
public static BuildScript DownloadFile(string address, string fileName, Action<Exception> exceptionCallback, ILogger logger) =>
Create(actions =>
{
if (actions.GetDirectoryName(fileName) is string dir && !string.IsNullOrWhiteSpace(dir))
actions.CreateDirectory(dir);
try
{
actions.DownloadFile(address, fileName);
actions.DownloadFile(address, fileName, logger);
return 0;
}
catch (Exception e)

43
csharp/extractor/Semmle.Util/FileUtils.cs
View File

@@ -4,6 +4,7 @@ using System.Linq;
using System.Net.Http;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Semmle.Util.Logging;
@@ -99,19 +100,49 @@ namespace Semmle.Util
return hex.ToString();
}
private static async Task DownloadFileAsync(string address, string filename)
private static async Task DownloadFileAsync(string address, string filename, HttpClient httpClient, CancellationToken token)
{
using var httpClient = new HttpClient();
using var contentStream = await httpClient.GetStreamAsync(address);
using var contentStream = await httpClient.GetStreamAsync(address, token);
using var stream = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None, 4096, true);
await contentStream.CopyToAsync(stream);
await contentStream.CopyToAsync(stream, CancellationToken.None);
}
private static void DownloadFileWithRetry(string address, string fileName, int tryCount, int timeoutMilliSeconds, ILogger logger)
{
logger.LogDebug($"Downloading {address} to {fileName}.");
using HttpClient client = new();
for (var i = 0; i < tryCount; i++)
{
logger.LogDebug($"Attempt {i + 1} of {tryCount}. Timeout: {timeoutMilliSeconds} ms.");
using var cts = new CancellationTokenSource();
cts.CancelAfter(timeoutMilliSeconds);
try
{
DownloadFileAsync(address, fileName, client, cts.Token).GetAwaiter().GetResult();
logger.LogDebug($"Downloaded {address} to {fileName}.");
return;
}
catch (Exception exc)
{
logger.LogDebug($"Failed to download {address} to {fileName}. Exception: {exc.Message}");
timeoutMilliSeconds *= 2;
if (i == tryCount - 1)
{
logger.LogDebug($"Failed to download {address} to {fileName} after {tryCount} attempts.");
// Rethrowing the last exception
throw;
}
}
}
}
/// <summary>
/// Downloads the file at <paramref name="address"/> to <paramref name="fileName"/>.
/// </summary>
public static void DownloadFile(string address, string fileName) =>
DownloadFileAsync(address, fileName).GetAwaiter().GetResult();
public static void DownloadFile(string address, string fileName, ILogger logger) =>
DownloadFileWithRetry(address, fileName, tryCount: 3, timeoutMilliSeconds: 10000, logger);
public static string ConvertPathToSafeRelativePath(string path)
{