hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-26 09:15:12 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15156 from tamasvajk/standalone/temp-folder-structure

Browse Source 
C#: Fix working directory structures in standalone
This commit is contained in:
Tamás Vajk
2023-12-20 11:57:42 +01:00
committed by GitHub
parent 44124158c4 b1413a1d98
commit d3c685a41a
9 changed files with 61 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs
View File

@@ -936,9 +936,8 @@ namespace Semmle.Autobuild.CSharp.Tests
{
actions.RunProcess["dotnet --list-sdks"] = 0;
actions.RunProcessOut["dotnet --list-sdks"] = "2.1.2 [C:\\Program Files\\dotnet\\sdks]\n2.1.4 [C:\\Program Files\\dotnet\\sdks]";
actions.RunProcess[@"chmod u+x dotnet-install.sh"] = 0;
actions.RunProcess[@"./dotnet-install.sh --channel release --version 2.1.3 --install-dir scratch/.dotnet"] = 0;
actions.RunProcess[@"rm dotnet-install.sh"] = 0;
actions.RunProcess[@"chmod u+x scratch/.dotnet/dotnet-install.sh"] = 0;
actions.RunProcess[@"scratch/.dotnet/dotnet-install.sh --channel release --version 2.1.3 --install-dir scratch/.dotnet"] = 0;
actions.RunProcess[@"scratch/.dotnet/dotnet --info"] = 0;
actions.RunProcess[@"scratch/.dotnet/dotnet clean C:\Project/test.csproj"] = 0;
actions.RunProcess[@"scratch/.dotnet/dotnet restore C:\Project/test.csproj"] = 0;
@@ -960,10 +959,11 @@ namespace Semmle.Autobuild.CSharp.Tests
</Project>");
actions.LoadXml[@"C:\Project/test.csproj"] = xml;
actions.DownloadFiles.Add(("https://dot.net/v1/dotnet-install.sh", "dotnet-install.sh"));
actions.DownloadFiles.Add(("https://dot.net/v1/dotnet-install.sh", "scratch/.dotnet/dotnet-install.sh"));
actions.CreateDirectories.Add(@"scratch/.dotnet");
var autobuilder = CreateAutoBuilder(false, dotnetVersion: "2.1.3");
TestAutobuilderScript(autobuilder, 0, 8);
TestAutobuilderScript(autobuilder, 0, 7);
}
[Fact]
@@ -972,9 +972,8 @@ namespace Semmle.Autobuild.CSharp.Tests
actions.RunProcess["dotnet --list-sdks"] = 0;
actions.RunProcessOut["dotnet --list-sdks"] = @"2.1.3 [C:\Program Files\dotnet\sdks]
2.1.4 [C:\Program Files\dotnet\sdks]";
actions.RunProcess[@"chmod u+x dotnet-install.sh"] = 0;
actions.RunProcess[@"./dotnet-install.sh --channel release --version 2.1.3 --install-dir scratch/.dotnet"] = 0;
actions.RunProcess[@"rm dotnet-install.sh"] = 0;
actions.RunProcess[@"chmod u+x scratch/.dotnet/dotnet-install.sh"] = 0;
actions.RunProcess[@"scratch/.dotnet/dotnet-install.sh --channel release --version 2.1.3 --install-dir scratch/.dotnet"] = 0;
actions.RunProcess[@"scratch/.dotnet/dotnet --info"] = 0;
actions.RunProcess[@"scratch/.dotnet/dotnet clean C:\Project/test.csproj"] = 0;
actions.RunProcess[@"scratch/.dotnet/dotnet restore C:\Project/test.csproj"] = 0;
@@ -996,10 +995,11 @@ namespace Semmle.Autobuild.CSharp.Tests
</Project>");
actions.LoadXml[@"C:\Project/test.csproj"] = xml;
actions.DownloadFiles.Add(("https://dot.net/v1/dotnet-install.sh", "dotnet-install.sh"));
actions.DownloadFiles.Add(("https://dot.net/v1/dotnet-install.sh", "scratch/.dotnet/dotnet-install.sh"));
actions.CreateDirectories.Add(@"scratch/.dotnet");
var autobuilder = CreateAutoBuilder(false, dotnetVersion: "2.1.3");
TestAutobuilderScript(autobuilder, 0, 8);
TestAutobuilderScript(autobuilder, 0, 7);
}
private void TestDotnetVersionWindows(Action action, int commandsRun)

25
csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs
View File

@@ -190,18 +190,23 @@ namespace Semmle.Autobuild.CSharp
}
else
{
var dotnetInstallPath = builder.Actions.PathCombine(FileUtils.GetTemporaryWorkingDirectory(
builder.Actions.GetEnvironmentVariable,
builder.Options.Language.UpperCaseName,
out var shouldCleanUp), ".dotnet", "dotnet-install.sh");
var downloadDotNetInstallSh = BuildScript.DownloadFile(
"https://dot.net/v1/dotnet-install.sh",
"dotnet-install.sh",
dotnetInstallPath,
e => builder.Log(Severity.Warning, $"Failed to download 'dotnet-install.sh': {e.Message}"));
var chmod = new CommandBuilder(builder.Actions).
RunCommand("chmod").
Argument("u+x").
Argument("dotnet-install.sh");
Argument(dotnetInstallPath);
var install = new CommandBuilder(builder.Actions).
RunCommand("./dotnet-install.sh").
RunCommand(dotnetInstallPath).
Argument("--channel").
Argument("release").
Argument("--version").
@@ -209,11 +214,17 @@ namespace Semmle.Autobuild.CSharp
Argument("--install-dir").
Argument(path);
var removeScript = new CommandBuilder(builder.Actions).
RunCommand("rm").
Argument("dotnet-install.sh");
var buildScript = downloadDotNetInstallSh & chmod.Script & install.Script;
return downloadDotNetInstallSh & chmod.Script & install.Script & BuildScript.Try(removeScript.Script);
if (shouldCleanUp)
{
var removeScript = new CommandBuilder(builder.Actions).
RunCommand("rm").
Argument(dotnetInstallPath);
buildScript &= removeScript.Script;
}
return buildScript;
}
});
}

8
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -52,7 +52,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
this.progressMonitor = new ProgressMonitor(logger);
this.sourceDir = new DirectoryInfo(srcDir);
packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "packages"));
legacyPackageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "legacypackages"));
missingPackageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "missingpackages"));
@@ -467,7 +467,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// with this source tree. Use a SHA1 of the directory name.
/// </summary>
/// <returns>The full path of the temp directory.</returns>
private static string ComputeTempDirectory(string srcDir, string packages = "packages")
private static string ComputeTempDirectory(string srcDir, string subfolderName)
{
var bytes = Encoding.Unicode.GetBytes(srcDir);
var sha = SHA1.HashData(bytes);
@@ -475,7 +475,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
foreach (var b in sha.Take(8))
sb.AppendFormat("{0:x2}", b);
return Path.Combine(FileUtils.GetTemporaryWorkingDirectory(out var _), "GitHub", packages, sb.ToString());
return Path.Combine(FileUtils.GetTemporaryWorkingDirectory(out var _), sb.ToString(), subfolderName);
}
/// <summary>
@@ -723,7 +723,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package =>
{
progressMonitor.NugetInstall(package);
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package));
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
var success = dotnet.New(tempDir.DirInfo.FullName);
if (!success)
{

22
csharp/extractor/Semmle.Util/FileUtils.cs
View File

@@ -144,20 +144,26 @@ namespace Semmle.Util
return nested;
}
private static readonly Lazy<string> tempFolderPath = new Lazy<string>(() =>
{
var tempPath = Path.GetTempPath();
var name = Guid.NewGuid().ToString("N").ToUpper();
var tempFolder = Path.Combine(tempPath, "GitHub", name);
Directory.CreateDirectory(tempFolder);
return tempFolder;
});
public static string GetTemporaryWorkingDirectory(Func<string, string?> getEnvironmentVariable, string lang, out bool shouldCleanUp)
{
shouldCleanUp = false;
var tempFolder = getEnvironmentVariable($"CODEQL_EXTRACTOR_{lang}_SCRATCH_DIR");
if (string.IsNullOrEmpty(tempFolder))
if (!string.IsNullOrEmpty(tempFolder))
{
var tempPath = Path.GetTempPath();
var name = Guid.NewGuid().ToString("N").ToUpper();
tempFolder = Path.Combine(tempPath, "GitHub", name);
shouldCleanUp = true;
shouldCleanUp = false;
return tempFolder;
}
return tempFolder;
shouldCleanUp = true;
return tempFolderPath.Value;
}
public static string GetTemporaryWorkingDirectory(out bool shouldCleanUp) =>

5
csharp/ql/integration-tests/all-platforms/standalone_dependencies_net48/Assemblies.ql
View File

@@ -4,10 +4,11 @@ private string getPath(Assembly a) {
not a.getCompilation().getOutputAssembly() = a and
exists(string s | s = a.getFile().getAbsolutePath() |
result =
s.substring(s.indexOf("GitHub/packages/") + "GitHub/packages/".length() + 16, s.length())
s.substring(s.indexOf("test-db/working/") + "test-db/working/".length() + 16 +
"/packages".length(), s.length())
or
result = s and
not exists(s.indexOf("GitHub/packages/"))
not exists(s.indexOf("test-db/working/"))
)
}

5
csharp/ql/integration-tests/posix-only/standalone_dependencies/Assemblies.ql
View File

@@ -4,10 +4,11 @@ private string getPath(Assembly a) {
not a.getCompilation().getOutputAssembly() = a and
exists(string s | s = a.getFile().getAbsolutePath() |
result =
s.substring(s.indexOf("GitHub/packages/") + "GitHub/packages/".length() + 16, s.length())
s.substring(s.indexOf("test-db/working/") + "test-db/working/".length() + 16 +
"/packages".length(), s.length())
or
result = s and
not exists(s.indexOf("GitHub/packages/"))
not exists(s.indexOf("test-db/working/"))
)
}

5
csharp/ql/integration-tests/posix-only/standalone_dependencies_multi_target/Assemblies.ql
View File

@@ -4,10 +4,11 @@ private string getPath(Assembly a) {
not a.getCompilation().getOutputAssembly() = a and
exists(string s | s = a.getFile().getAbsolutePath() |
result =
s.substring(s.indexOf("GitHub/packages/") + "GitHub/packages/".length() + 16, s.length())
s.substring(s.indexOf("test-db/working/") + "test-db/working/".length() + 16 +
"/packages".length(), s.length())
or
result = s and
not exists(s.indexOf("GitHub/packages/"))
not exists(s.indexOf("test-db/working/"))
)
}

16
csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget/Assemblies.ql
View File

@@ -4,19 +4,9 @@ private string getPath(Assembly a) {
not a.getCompilation().getOutputAssembly() = a and
exists(string s | s = a.getFile().getAbsolutePath() |
result =
s.substring(s.indexOf("GitHub/packages/") + "GitHub/packages/".length() + 16, s.length())
or
result =
s.substring(s.indexOf("GitHub/legacypackages/") + "GitHub/legacypackages/".length() + 16,
s.length())
// TODO: excluding all other assemblies from the test result as mono installations seem problematic on ARM runners.
// or
// result = s.substring(s.indexOf("lib/mono/") + "lib/mono/".length(), s.length())
// or
// result = s and
// not exists(s.indexOf("GitHub/packages/")) and
// not exists(s.indexOf("GitHub/legacypackages/")) and
// not exists(s.indexOf("lib/mono/"))
s.substring(s.indexOf("test-db/working/") + "test-db/working/".length() + 16 +
"/legacypackages".length(), s.length())
// TODO: include all other assemblies from the test results. Initially disable because mono installations were problematic on ARM runners.
)
}

5
csharp/ql/integration-tests/windows-only/standalone_dependencies/Assemblies.ql
View File

@@ -4,10 +4,11 @@ private string getPath(Assembly a) {
not a.getCompilation().getOutputAssembly() = a and
exists(string s | s = a.getFile().getAbsolutePath() |
result =
s.substring(s.indexOf("GitHub/packages/") + "GitHub/packages/".length() + 16, s.length())
s.substring(s.indexOf("test-db/working/") + "test-db/working/".length() + 16 +
"/packages".length(), s.length())
or
result = s and
not exists(s.indexOf("GitHub/packages/"))
not exists(s.indexOf("test-db/working/"))
)
}