hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-21 15:05:56 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15577 from tamasvajk/feature/missing-nuget-sources

Browse Source 
C# - Add default nuget feed if there's none
This commit is contained in:
Tamás Vajk
2024-02-13 09:18:32 +01:00
committed by GitHub
parent faefa056eb 888f47c6c8
commit de13ff6afd
28 changed files with 195 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -169,13 +169,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
try
{
var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger);
var count = nuget.InstallPackages();
if (nuget.PackageCount > 0)
using (var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger))
{
CompilationInfos.Add(("packages.config files", nuget.PackageCount.ToString()));
CompilationInfos.Add(("Successfully restored packages.config files", count.ToString()));
var count = nuget.InstallPackages();
if (nuget.PackageCount > 0)
{
CompilationInfos.Add(("packages.config files", nuget.PackageCount.ToString()));
CompilationInfos.Add(("Successfully restored packages.config files", count.ToString()));
}
}
var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });

130
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs
View File

@@ -1,8 +1,8 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using Microsoft.Build.Framework;
using Semmle.Util;
namespace Semmle.Extraction.CSharp.DependencyFetching
@@ -12,7 +12,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// Locates packages in a source tree and downloads all of the
/// referenced assemblies to a temp folder.
/// </summary>
internal class NugetPackages
internal class NugetPackages : IDisposable
{
private readonly string? nugetExe;
private readonly Util.Logging.ILogger logger;
@@ -24,6 +24,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public int PackageCount => packageFiles.Length;
private readonly string? backupNugetConfig;
private readonly string? nugetConfigPath;
/// <summary>
/// The computed packages directory.
/// This will be in the Temp location
@@ -47,6 +50,41 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
logger.LogInfo($"Found {packageFiles.Length} packages.config files, trying to use nuget.exe for package restore");
nugetExe = ResolveNugetExe(sourceDir);
if (HasNoPackageSource())
{
// We only modify or add a top level nuget.config file
nugetConfigPath = Path.Combine(sourceDir, "nuget.config");
try
{
if (File.Exists(nugetConfigPath))
{
var tempFolderPath = FileUtils.GetTemporaryWorkingDirectory(out var _);
do
{
backupNugetConfig = Path.Combine(tempFolderPath, Path.GetRandomFileName());
}
while (File.Exists(backupNugetConfig));
File.Copy(nugetConfigPath, backupNugetConfig, true);
}
else
{
File.WriteAllText(nugetConfigPath,
"""
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<packageSources>
</packageSources>
</configuration>
""");
}
AddDefaultPackageSource(nugetConfigPath);
}
catch (Exception e)
{
logger.LogError($"Failed to add default package source to {nugetConfigPath}: {e}");
}
}
}
else
{
@@ -118,15 +156,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
*/
string exe, args;
if (Util.Win32.IsWindows())
if (Win32.IsWindows())
{
exe = nugetExe!;
args = string.Format("install -OutputDirectory {0} {1}", packageDirectory, package);
args = $"install -OutputDirectory {packageDirectory} {package}";
}
else
{
exe = "mono";
args = string.Format("{0} install -OutputDirectory {1} {2}", nugetExe, packageDirectory, package);
args = $"{nugetExe} install -OutputDirectory {packageDirectory} {package}";
}
var pi = new ProcessStartInfo(exe, args)
@@ -159,5 +197,87 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
return packageFiles.Count(package => TryRestoreNugetPackage(package.FullName));
}
private bool HasNoPackageSource()
{
if (Win32.IsWindows())
{
return false;
}
try
{
logger.LogInfo("Checking if default package source is available...");
RunMonoNugetCommand("sources list -ForceEnglishOutput", out var stdout);
if (stdout.All(line => line != "No sources found."))
{
return false;
}
return true;
}
catch (Exception e)
{
logger.LogWarning($"Failed to check if default package source is added: {e}");
return false;
}
}
private void RunMonoNugetCommand(string command, out IList<string> stdout)
{
var exe = "mono";
var args = $"{nugetExe} {command}";
var pi = new ProcessStartInfo(exe, args)
{
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false
};
var threadId = Environment.CurrentManagedThreadId;
void onOut(string s) => logger.LogInfo(s, threadId);
void onError(string s) => logger.LogError(s, threadId);
pi.ReadOutput(out stdout, onOut, onError);
}
private void AddDefaultPackageSource(string nugetConfig)
{
logger.LogInfo("Adding default package source...");
RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source https://api.nuget.org/v3/index.json -ConfigFile \"{nugetConfig}\"", out var _);
}
public void Dispose()
{
if (nugetConfigPath is null)
{
return;
}
try
{
if (backupNugetConfig is null)
{
logger.LogInfo("Removing nuget.config file");
File.Delete(nugetConfigPath);
return;
}
logger.LogInfo("Reverting nuget.config file content");
// The content of the original nuget.config file is reverted without changing the file's attributes or casing:
using (var backup = File.OpenRead(backupNugetConfig))
using (var current = File.OpenWrite(nugetConfigPath))
{
current.SetLength(0); // Truncate file
backup.CopyTo(current); // Restore original content
}
logger.LogInfo("Deleting backup nuget.config file");
File.Delete(backupNugetConfig);
}
catch (Exception exc)
{
logger.LogError($"Failed to restore original nuget.config file: {exc}");
}
}
}
}

2
csharp/ql/integration-tests/all-platforms/autobuild/test.py
View File

@@ -1,3 +1,3 @@
from create_database_utils import *
run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=cil=false"])
run_codeql_database_create([], lang="csharp")

2
csharp/ql/integration-tests/all-platforms/cshtml/test.py
View File

@@ -1,3 +1,3 @@
from create_database_utils import *
run_codeql_database_create(['dotnet build'], lang="csharp", extra_args=["--extractor-option=cil=false"])
run_codeql_database_create(['dotnet build'], lang="csharp")

2
csharp/ql/integration-tests/all-platforms/cshtml_standalone/test.py
View File

@@ -1,4 +1,4 @@
import os
from create_database_utils import *
run_codeql_database_create(lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
run_codeql_database_create(lang="csharp", extra_args=["--extractor-option=buildless=true"])

2
csharp/ql/integration-tests/all-platforms/cshtml_standalone_disabled/test.py
View File

@@ -2,4 +2,4 @@ import os
from create_database_utils import *
os.environ['CODEQL_EXTRACTOR_CSHARP_STANDALONE_EXTRACT_WEB_VIEWS'] = 'false'
run_codeql_database_create(lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
run_codeql_database_create(lang="csharp", extra_args=["--extractor-option=buildless=true"])

2
csharp/ql/integration-tests/all-platforms/cshtml_standalone_flowsteps/test.py
View File

@@ -1,4 +1,4 @@
import os
from create_database_utils import *
run_codeql_database_create(lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
run_codeql_database_create(lang="csharp", extra_args=["--extractor-option=buildless=true"])

2
csharp/ql/integration-tests/all-platforms/cshtml_standalone_net6/test.py
View File

@@ -1,4 +1,4 @@
import os
from create_database_utils import *
run_codeql_database_create(lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
run_codeql_database_create(lang="csharp", extra_args=["--extractor-option=buildless=true"])

2
csharp/ql/integration-tests/all-platforms/diag_recursive_generics/test.py
View File

@@ -1,3 +1,3 @@
from create_database_utils import *
run_codeql_database_create(['dotnet build'], lang="csharp", extra_args=["--extractor-option=cil=false"])
run_codeql_database_create(['dotnet build'], lang="csharp")

2
csharp/ql/integration-tests/all-platforms/standalone/test.py
View File

@@ -1,3 +1,3 @@
from create_database_utils import *
run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true"])

2
csharp/ql/integration-tests/all-platforms/standalone_dependencies_net48/test.py
View File

@@ -1,3 +1,3 @@
from create_database_utils import *
run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true"])

2
csharp/ql/integration-tests/linux-only/compiler_args/test.py
View File

@@ -1,3 +1,3 @@
from create_database_utils import *
run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=cil=false"])
run_codeql_database_create([], lang="csharp")

2
csharp/ql/integration-tests/linux-only/standalone_dependencies_non_utf8_filename/test.py
View File

@@ -5,4 +5,4 @@ path = b'\xd2abcd.cs'
with open(path, 'w') as file:
file.write('class X { }\n')
run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true"])

2
csharp/ql/integration-tests/posix-only/standalone_dependencies/test.py
View File

@@ -1,3 +1,3 @@
from create_database_utils import *
run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true"])

2
csharp/ql/integration-tests/posix-only/standalone_dependencies_multi_target/test.py
View File

@@ -1,3 +1,3 @@
from create_database_utils import *
run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true"])

2
csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget/test.py
View File

@@ -1,3 +1,3 @@
from create_database_utils import *
run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true"])

2
csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/test.py
View File

@@ -1,3 +1,3 @@
from create_database_utils import *
run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true"])

1
csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/Assemblies.expected Normal file
View File

@@ -0,0 +1 @@
| /Newtonsoft.Json.6.0.4/lib/portable-net45+wp80+win8+wpa81/Newtonsoft.Json.dll |

14
csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/Assemblies.ql Normal file
View File

@@ -0,0 +1,14 @@
import csharp
private string getPath(Assembly a) {
not a.getCompilation().getOutputAssembly() = a and
exists(string s | s = a.getFile().getAbsolutePath() |
result =
s.substring(s.indexOf("test-db/working/") + "test-db/working/".length() + 16 +
"/legacypackages".length(), s.length())
// TODO: include all other assemblies from the test results. Initially disable because mono installations were problematic on ARM runners.
)
}
from Assembly a
select getPath(a)

7
csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/nuget.config Normal file
View File

@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<packageSources>
<!-- Remove all nuget package sources to force adding the default feed in the extraction root. -->
<clear />
</packageSources>
</configuration>

6
csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/proj/Program.cs Normal file
View File

@@ -0,0 +1,6 @@
class Program
{
static void Main(string[] args)
{
}
}

5
csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/proj/global.json Normal file
View File

@@ -0,0 +1,5 @@
{
"sdk": {
"version": "8.0.101"
}
}

4
csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/proj/packages.config Normal file
View File

@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Newtonsoft.Json" version="6.0.4" targetFramework="net461" />
</packages>

4
csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/proj/test.csproj Normal file
View File

@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
</Project>

1
csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/skip-on-platform-osx-arm Normal file
View File

@@ -0,0 +1 @@
Skipping the test on the ARM runners, as we're running into trouble with Mono and nuget.

3
csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/test.py Normal file
View File

@@ -0,0 +1,3 @@
from create_database_utils import *
run_codeql_database_create([], source="proj", lang="csharp", extra_args=["--extractor-option=buildless=true"])

2
csharp/ql/integration-tests/posix-only/warn_as_error/test.py
View File

@@ -2,6 +2,6 @@ import os
from create_database_utils import *
from diagnostics_test_utils import *
run_codeql_database_create(["./build.sh"], lang="csharp", extra_args=["--extractor-option=cil=false"])
run_codeql_database_create(["./build.sh"], lang="csharp")
check_diagnostics()

2
csharp/ql/integration-tests/windows-only/standalone_dependencies/test.py
View File

@@ -1,3 +1,3 @@
from create_database_utils import *
run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true"])