hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-27 17:55:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

C#: Extract compilation DB entity in standalone mode

Browse Source
This commit is contained in:
Tamas Vajk
2023-10-02 12:48:58 +02:00
parent 5dccc8d33e
commit de45a9b137
16 changed files with 120 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -60,7 +60,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
this.progressMonitor.FindingFiles(srcDir);
packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
tempWorkingDirectory = new TemporaryDirectory(GetTemporaryWorkingDirectory(out cleanupTempWorkingDirectory));
tempWorkingDirectory = new TemporaryDirectory(FileUtils.GetTemporaryWorkingDirectory(out cleanupTempWorkingDirectory));
var allFiles = GetAllFiles();
var binaryFileExtensions = new HashSet<string>(new[] { ".dll", ".exe" }); // TODO: add more binary file extensions.
@@ -286,22 +286,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return Path.Combine(Path.GetTempPath(), "GitHub", "packages", sb.ToString());
}
private static string GetTemporaryWorkingDirectory(out bool cleanupTempWorkingDirectory)
{
cleanupTempWorkingDirectory = false;
var tempFolder = EnvironmentVariables.GetScratchDirectory();
if (string.IsNullOrEmpty(tempFolder))
{
var tempPath = Path.GetTempPath();
var name = Guid.NewGuid().ToString("N").ToUpper();
tempFolder = Path.Combine(tempPath, "GitHub", name);
cleanupTempWorkingDirectory = true;
}
return tempFolder;
}
/// <summary>
/// Creates a temporary directory with the given subfolder name.
/// The created directory might be inside the repo folder, and it is deleted when the object is disposed.

57
csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs
View File

@@ -30,29 +30,46 @@ namespace Semmle.Extraction.CSharp.Standalone
IProgressMonitor progressMonitor,
Stopwatch stopwatch)
{
CSharp.Extractor.Analyse(stopwatch, analyser, options,
references => GetResolvedReferencesStandalone(referencePaths, references),
(analyser, syntaxTrees) => CSharp.Extractor.ReadSyntaxTrees(sources, analyser, null, null, syntaxTrees),
(syntaxTrees, references) => CSharpCompilation.Create(
"csharp.dll", syntaxTrees, references, new CSharpCompilationOptions(OutputKind.ConsoleApplication, allowUnsafe: true)
),
(compilation, options) => analyser.Initialize(compilation, options),
() => { },
_ => { },
() =>
var output = FileUtils.CreateTemporaryFile(".dll", out var shouldCleanUpContainingFolder);
try
{
CSharp.Extractor.Analyse(stopwatch, analyser, options,
references => GetResolvedReferencesStandalone(referencePaths, references),
(analyser, syntaxTrees) => CSharp.Extractor.ReadSyntaxTrees(sources, analyser, null, null, syntaxTrees),
(syntaxTrees, references) => CSharpCompilation.Create(
output.Name, syntaxTrees, references, new CSharpCompilationOptions(OutputKind.ConsoleApplication, allowUnsafe: true)
),
(compilation, options) => analyser.Initialize(output.FullName, compilation, options),
_ => { },
() =>
{
foreach (var type in analyser.MissingNamespaces)
{
progressMonitor.MissingNamespace(type);
}
foreach (var type in analyser.MissingTypes)
{
progressMonitor.MissingType(type);
}
progressMonitor.MissingSummary(analyser.MissingTypes.Count(), analyser.MissingNamespaces.Count());
});
}
finally
{
try
{
foreach (var type in analyser.MissingNamespaces)
FileUtils.TryDelete(output.FullName);
if (shouldCleanUpContainingFolder)
{
progressMonitor.MissingNamespace(type);
output.Directory?.Delete(true);
}
foreach (var type in analyser.MissingTypes)
{
progressMonitor.MissingType(type);
}
progressMonitor.MissingSummary(analyser.MissingTypes.Count(), analyser.MissingNamespaces.Count());
});
}
catch
{ }
}
}
private static void ExtractStandalone(

4
csharp/extractor/Semmle.Extraction.CSharp/Entities/Assembly.cs
View File

@@ -16,7 +16,7 @@ namespace Semmle.Extraction.CSharp.Entities
if (init is null)
{
// This is the output assembly
assemblyPath = ((TracingExtractor)cx.Extractor).OutputPath;
assemblyPath = cx.Extractor.OutputPath;
assembly = cx.Compilation.Assembly;
}
else
@@ -63,8 +63,6 @@ namespace Semmle.Extraction.CSharp.Entities
public static Assembly CreateOutputAssembly(Context cx)
{
if (cx.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
throw new InternalError("Attempting to create the output assembly in standalone extraction mode");
return AssemblyConstructorFactory.Instance.CreateEntity(cx, outputAssemblyCacheKey, null);
}

5
csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs
View File

@@ -63,10 +63,7 @@ namespace Semmle.Extraction.CSharp.Entities
if (attributeSyntax is not null)
{
if (!Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
{
trapFile.attribute_location(this, Assembly.CreateOutputAssembly(Context));
}
trapFile.attribute_location(this, Assembly.CreateOutputAssembly(Context));
TypeMention.Create(Context, attributeSyntax.Name, this, type);
}

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/CachedSymbol.cs
View File

@@ -97,7 +97,7 @@ namespace Semmle.Extraction.CSharp.Entities
{
// Some built in operators lack locations, so loc is null.
yield return Context.CreateLocation(ReportingLocation);
if (!Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone) && loc.Kind == LocationKind.SourceFile)
if (loc.Kind == LocationKind.SourceFile)
yield return Assembly.CreateOutputAssembly(Context);
}
}

7
csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/PreprocessorDirective.cs
View File

@@ -15,11 +15,8 @@ namespace Semmle.Extraction.CSharp.Entities
trapFile.preprocessor_directive_active(this, Symbol.IsActive);
trapFile.preprocessor_directive_location(this, Context.CreateLocation(ReportingLocation));
if (!Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
{
var compilation = Compilation.Create(Context);
trapFile.preprocessor_directive_compilation(this, compilation);
}
var compilation = Compilation.Create(Context);
trapFile.preprocessor_directive_compilation(this, compilation);
}
protected abstract void PopulatePreprocessor(TextWriter trapFile);

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs
View File

@@ -108,7 +108,7 @@ namespace Semmle.Extraction.CSharp.Entities
foreach (var l in GetLocations(Symbol))
yield return Context.CreateLocation(l);
if (!Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone) && Symbol.DeclaringSyntaxReferences.Any())
if (Symbol.DeclaringSyntaxReferences.Any())
yield return Assembly.CreateOutputAssembly(Context);
}
}

31
csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs
View File

@@ -19,6 +19,8 @@ namespace Semmle.Extraction.CSharp
protected Extraction.Extractor? extractor;
protected CSharpCompilation? compilation;
protected CommonOptions? options;
private protected Entities.Compilation? compilationEntity;
private IDisposable? compilationTrapFile;
private readonly object progressMutex = new object();
@@ -226,8 +228,35 @@ namespace Semmle.Extraction.CSharp
}
}
private void DoAnalyseCompilation()
{
try
{
var assemblyPath = extractor.OutputPath;
var transformedAssemblyPath = PathTransformer.Transform(assemblyPath);
var assembly = compilation.Assembly;
var trapWriter = transformedAssemblyPath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
compilationTrapFile = trapWriter; // Dispose later
var cx = new Context(extractor, compilation.Clone(), trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
compilationEntity = Entities.Compilation.Create(cx);
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
Logger.Log(Severity.Error, " Unhandled exception analyzing {0}: {1}", "compilation", ex);
}
}
#nullable restore warnings
/// <summary>
/// Extracts compilation-wide entities, such as compilations and compiler diagnostics.
/// </summary>
public void AnalyseCompilation()
{
extractionTasks.Add(() => DoAnalyseCompilation());
}
private static bool FileIsUpToDate(string src, string dest)
{
return File.Exists(dest) &&
@@ -275,6 +304,8 @@ namespace Semmle.Extraction.CSharp
Logger.Log(Severity.Info, "EXTRACTION SUCCEEDED in {0}", stopWatch.Elapsed);
Logger.Dispose();
compilationTrapFile?.Dispose();
}
/// <summary>

4
csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs
View File

@@ -302,7 +302,6 @@ namespace Semmle.Extraction.CSharp
Func<Analyser, List<SyntaxTree>, IEnumerable<Action>> getSyntaxTreeTasks,
Func<IEnumerable<SyntaxTree>, IEnumerable<MetadataReference>, CSharpCompilation> getCompilation,
Action<CSharpCompilation, CommonOptions> initializeAnalyser,
Action analyseCompilation,
Action<Entities.PerformanceMetrics> logPerformance,
Action postProcess)
{
@@ -332,7 +331,7 @@ namespace Semmle.Extraction.CSharp
var compilation = getCompilation(syntaxTrees, references);
initializeAnalyser(compilation, options);
analyseCompilation();
analyser.AnalyseCompilation();
analyser.AnalyseReferences();
foreach (var tree in compilation.SyntaxTrees)
@@ -416,7 +415,6 @@ namespace Semmle.Extraction.CSharp
);
},
(compilation, options) => analyser.EndInitialize(compilerArguments, options, compilation),
() => analyser.AnalyseCompilation(),
performance => analyser.LogPerformance(performance),
() => { });
}

8
csharp/extractor/Semmle.Extraction.CSharp.Standalone/StandaloneAnalyser.cs → csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs
View File

@@ -1,4 +1,6 @@
using System;
using System.Collections.Generic;
using System.IO;
using Microsoft.CodeAnalysis.CSharp;
using Semmle.Util.Logging;
@@ -11,13 +13,15 @@ namespace Semmle.Extraction.CSharp
{
}
public void Initialize(CSharpCompilation compilationIn, CommonOptions options)
public void Initialize(string outputPath, CSharpCompilation compilationIn, CommonOptions options)
{
compilation = compilationIn;
extractor = new StandaloneExtractor(Logger, PathTransformer, options);
extractor = new StandaloneExtractor(outputPath, Logger, PathTransformer, options);
this.options = options;
LogExtractorInfo(Extraction.Extractor.Version);
SetReferencePaths();
Entities.Compilation.Settings = (Directory.GetCurrentDirectory(), Array.Empty<string>());
}
#nullable disable warnings

38
csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs
View File

@@ -9,11 +9,8 @@ using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp
{
public class TracingAnalyser : Analyser, IDisposable
public class TracingAnalyser : Analyser
{
private Entities.Compilation? compilationEntity;
private IDisposable? compilationTrapFile;
private bool init;
public TracingAnalyser(IProgressMonitor pm, ILogger logger, bool addAssemblyTrapPrefix, PathTransformer pathTransformer)
@@ -55,20 +52,6 @@ namespace Semmle.Extraction.CSharp
CompilationErrors += FilteredDiagnostics.Count();
}
public override void Dispose()
{
compilationTrapFile?.Dispose();
base.Dispose();
}
/// <summary>
/// Extracts compilation-wide entities, such as compilations and compiler diagnostics.
/// </summary>
public void AnalyseCompilation()
{
extractionTasks.Add(() => DoAnalyseCompilation());
}
/// <summary>
/// Logs information about the extractor, as well as the arguments to Roslyn.
/// </summary>
@@ -193,25 +176,6 @@ namespace Semmle.Extraction.CSharp
}
}
private void DoAnalyseCompilation()
{
try
{
var assemblyPath = ((TracingExtractor?)extractor).OutputPath;
var transformedAssemblyPath = PathTransformer.Transform(assemblyPath);
var assembly = compilation.Assembly;
var trapWriter = transformedAssemblyPath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
compilationTrapFile = trapWriter; // Dispose later
var cx = new Context(extractor, compilation.Clone(), trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
compilationEntity = Entities.Compilation.Create(cx);
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
Logger.Log(Severity.Error, " Unhandled exception analyzing {0}: {1}", "compilation", ex);
}
}
public void LogPerformance(Entities.PerformanceMetrics p) => compilationEntity.PopulatePerformance(p);
#nullable restore warnings

3
csharp/extractor/Semmle.Extraction.CSharp/Populators/TypeContainerVisitor.cs
View File

@@ -82,9 +82,6 @@ namespace Semmle.Extraction.CSharp.Populators
public override void VisitAttributeList(AttributeListSyntax node)
{
if (Cx.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
return;
var outputAssembly = Assembly.CreateOutputAssembly(Cx);
var kind = node.Target?.Identifier.Kind() switch
{

4
csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs
View File

@@ -9,14 +9,16 @@ namespace Semmle.Extraction
public abstract class Extractor
{
public abstract ExtractorMode Mode { get; }
public string OutputPath { get; }
/// <summary>
/// Creates a new extractor instance for one compilation unit.
/// </summary>
/// <param name="logger">The object used for logging.</param>
/// <param name="pathTransformer">The object used for path transformations.</param>
protected Extractor(ILogger logger, PathTransformer pathTransformer)
protected Extractor(string outputPath, ILogger logger, PathTransformer pathTransformer)
{
OutputPath = outputPath;
Logger = logger;
PathTransformer = pathTransformer;
}

2
csharp/extractor/Semmle.Extraction/Extractor/StandaloneExtractor.cs
View File

@@ -11,7 +11,7 @@ namespace Semmle.Extraction
/// </summary>
/// <param name="logger">The object used for logging.</param>
/// <param name="pathTransformer">The object used for path transformations.</param>
public StandaloneExtractor(ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(logger, pathTransformer)
public StandaloneExtractor(string outputPath, ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(outputPath, logger, pathTransformer)
{
Mode = ExtractorMode.Standalone;
if (options.QlTest)

4
csharp/extractor/Semmle.Extraction/Extractor/TracingExtractor.cs
View File

@@ -5,7 +5,6 @@ namespace Semmle.Extraction
public class TracingExtractor : Extractor
{
public override ExtractorMode Mode { get; }
public string OutputPath { get; }
/// <summary>
/// Creates a new extractor instance for one compilation unit.
@@ -13,9 +12,8 @@ namespace Semmle.Extraction
/// <param name="outputPath">The name of the output DLL/EXE, or null if not specified (standalone extraction).</param>
/// <param name="logger">The object used for logging.</param>
/// <param name="pathTransformer">The object used for path transformations.</param>
public TracingExtractor(string outputPath, ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(logger, pathTransformer)
public TracingExtractor(string outputPath, ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(outputPath, logger, pathTransformer)
{
OutputPath = outputPath;
Mode = ExtractorMode.None;
if (options.QlTest)
{

32
csharp/extractor/Semmle.Util/FileUtils.cs
View File

@@ -143,5 +143,37 @@ namespace Semmle.Util
}
return nested;
}
public static string GetTemporaryWorkingDirectory(out bool shouldCleanUp)
{
shouldCleanUp = false;
var tempFolder = EnvironmentVariables.GetScratchDirectory();
if (string.IsNullOrEmpty(tempFolder))
{
var tempPath = Path.GetTempPath();
var name = Guid.NewGuid().ToString("N").ToUpper();
tempFolder = Path.Combine(tempPath, "GitHub", name);
shouldCleanUp = true;
}
return tempFolder;
}
public static FileInfo CreateTemporaryFile(string extension, out bool shouldCleanUpContainingFolder)
{
var tempFolder = GetTemporaryWorkingDirectory(out shouldCleanUpContainingFolder);
Directory.CreateDirectory(tempFolder);
string outputPath;
do
{
outputPath = Path.Combine(tempFolder, Path.GetRandomFileName() + extension);
}
while (File.Exists(outputPath));
File.Create(outputPath);
return new FileInfo(outputPath);
}
}
}