hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-25 00:35:20 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #16732 from tamasvajk/refactor/extraction-states

Browse Source 
C#: Refactor extractor state classes and simplify extraction code
This commit is contained in:
Tamás Vajk
2024-06-12 15:20:11 +02:00
committed by GitHub
parent 4ee80653e2 0df6a1c4b1
commit a756f86847
25 changed files with 163 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs
View File

@@ -42,17 +42,17 @@ namespace Semmle.Extraction.CSharp.Standalone
(compilation, options) => analyser.Initialize(output.FullName, extractionInput.CompilationInfos, compilation, options),
() =>
{
foreach (var type in analyser.MissingNamespaces)
foreach (var type in analyser.ExtractionContext!.MissingNamespaces)
{
progressMonitor.MissingNamespace(type);
}
foreach (var type in analyser.MissingTypes)
foreach (var type in analyser.ExtractionContext!.MissingTypes)
{
progressMonitor.MissingType(type);
}
progressMonitor.MissingSummary(analyser.MissingTypes.Count(), analyser.MissingNamespaces.Count());
progressMonitor.MissingSummary(analyser.ExtractionContext!.MissingTypes.Count(), analyser.ExtractionContext!.MissingNamespaces.Count());
});
}
finally
@@ -69,29 +69,6 @@ namespace Semmle.Extraction.CSharp.Standalone
}
}
private static void ExtractStandalone(
ExtractionInput extractionInput,
IProgressMonitor pm,
ILogger logger,
CommonOptions options)
{
var stopwatch = new Stopwatch();
stopwatch.Start();
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
var pathTransformer = new PathTransformer(canonicalPathCache);
using var analyser = new StandaloneAnalyser(pm, logger, false, pathTransformer);
try
{
AnalyseStandalone(analyser, extractionInput, options, pm, stopwatch);
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
analyser.Logger.Log(Severity.Error, " Unhandled exception: {0}", ex);
}
}
private class ExtractionProgress : IProgressMonitor
{
public ExtractionProgress(ILogger output)
@@ -141,8 +118,8 @@ namespace Semmle.Extraction.CSharp.Standalone
public static ExitCode Run(Options options)
{
var stopwatch = new Stopwatch();
stopwatch.Start();
var overallStopwatch = new Stopwatch();
overallStopwatch.Start();
using var logger = new ConsoleLogger(options.Verbosity, logThreadId: true);
logger.Log(Severity.Info, "Extracting C# with build-mode set to 'none'");
@@ -158,12 +135,26 @@ namespace Semmle.Extraction.CSharp.Standalone
logger.Log(Severity.Info, "");
logger.Log(Severity.Info, "Extracting...");
ExtractStandalone(
new ExtractionInput(dependencyManager.AllSourceFiles, dependencyManager.ReferenceFiles, dependencyManager.CompilationInfos),
new ExtractionProgress(logger),
fileLogger,
options);
logger.Log(Severity.Info, $"Extraction completed in {stopwatch.Elapsed}");
var analyzerStopwatch = new Stopwatch();
analyzerStopwatch.Start();
var canonicalPathCache = CanonicalPathCache.Create(fileLogger, 1000);
var pathTransformer = new PathTransformer(canonicalPathCache);
var progressMonitor = new ExtractionProgress(logger);
using var analyser = new StandaloneAnalyser(progressMonitor, fileLogger, pathTransformer, canonicalPathCache, false);
try
{
var extractionInput = new ExtractionInput(dependencyManager.AllSourceFiles, dependencyManager.ReferenceFiles, dependencyManager.CompilationInfos);
AnalyseStandalone(analyser, extractionInput, options, progressMonitor, analyzerStopwatch);
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
fileLogger.Log(Severity.Error, " Unhandled exception: {0}", ex);
}
logger.Log(Severity.Info, $"Extraction completed in {overallStopwatch.Elapsed}");
return ExitCode.Ok;
}

8
csharp/extractor/Semmle.Extraction.CSharp/Entities/Assembly.cs
View File

@@ -17,7 +17,7 @@ namespace Semmle.Extraction.CSharp.Entities
isOutputAssembly = init is null;
if (isOutputAssembly)
{
assemblyPath = cx.Extractor.OutputPath;
assemblyPath = cx.ExtractionContext.OutputPath;
assembly = cx.Compilation.Assembly;
}
else
@@ -25,7 +25,7 @@ namespace Semmle.Extraction.CSharp.Entities
assembly = init!.MetadataModule!.ContainingAssembly;
var identity = assembly.Identity;
var idString = identity.Name + " " + identity.Version;
assemblyPath = cx.Extractor.GetAssemblyFile(idString);
assemblyPath = cx.ExtractionContext.GetAssemblyFile(idString);
}
}
@@ -33,7 +33,7 @@ namespace Semmle.Extraction.CSharp.Entities
{
if (assemblyPath is not null)
{
var isBuildlessOutputAssembly = isOutputAssembly && Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone);
var isBuildlessOutputAssembly = isOutputAssembly && Context.ExtractionContext.Mode.HasFlag(ExtractorMode.Standalone);
var identifier = isBuildlessOutputAssembly
? ""
: assembly.ToString() ?? "";
@@ -74,7 +74,7 @@ namespace Semmle.Extraction.CSharp.Entities
public override void WriteId(EscapingTextWriter trapFile)
{
if (isOutputAssembly && Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
if (isOutputAssembly && Context.ExtractionContext.Mode.HasFlag(ExtractorMode.Standalone))
{
trapFile.Write("buildlessOutputAssembly");
}

4
csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs
View File

@@ -18,8 +18,8 @@ namespace Semmle.Extraction.CSharp.Entities
#nullable disable warnings
private Compilation(Context cx) : base(cx, null)
{
cwd = cx.Extractor.Cwd;
args = cx.Extractor.Args;
cwd = cx.ExtractionContext.Cwd;
args = cx.ExtractionContext.Args;
hashCode = cwd.GetHashCode();
for (var i = 0; i < args.Length; i++)
{

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/CompilerDiagnostic.cs
View File

@@ -28,7 +28,7 @@ namespace Semmle.Extraction.CSharp.Entities
{
if (messageCount == limit + 1)
{
Context.Extractor.Logger.LogWarning($"Stopped logging {key} compiler diagnostics for the current compilation after reaching {limit}");
Context.ExtractionContext.Logger.LogWarning($"Stopped logging {key} compiler diagnostics for the current compilation after reaching {limit}");
}
return;

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Invocation.cs
View File

@@ -133,7 +133,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
.Where(method => method.Parameters.Length >= Syntax.ArgumentList.Arguments.Count)
.Where(method => method.Parameters.Count(p => !p.HasExplicitDefaultValue) <= Syntax.ArgumentList.Arguments.Count);
return Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone) ?
return Context.ExtractionContext.Mode.HasFlag(ExtractorMode.Standalone) ?
candidates.FirstOrDefault() :
candidates.SingleOrDefault();
}

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/File.cs
View File

@@ -61,7 +61,7 @@ namespace Semmle.Extraction.CSharp.Entities
}
}
trapFile.file_extraction_mode(this, Context.Extractor.Mode);
trapFile.file_extraction_mode(this, Context.ExtractionContext.Mode);
}
private bool IsPossiblyTextFile()

22
csharp/extractor/Semmle.Extraction.CSharp/Entities/NonGeneratedSourceLocation.cs
View File

@@ -27,7 +27,7 @@ namespace Semmle.Extraction.CSharp.Entities
var mapped = Symbol.GetMappedLineSpan();
if (mapped.HasMappedPath && mapped.IsValid)
{
var path = TryAdjustRelativeMappedFilePath(mapped.Path, Position.Path, Context.Extractor.Logger);
var path = Context.TryAdjustRelativeMappedFilePath(mapped.Path, Position.Path);
var mappedLoc = Create(Context, Location.Create(path, default, mapped.Span));
trapFile.locations_mapped(this, mappedLoc);
@@ -64,25 +64,5 @@ namespace Semmle.Extraction.CSharp.Entities
public override NonGeneratedSourceLocation Create(Context cx, Location init) => new NonGeneratedSourceLocation(cx, init);
}
public static string TryAdjustRelativeMappedFilePath(string mappedToPath, string mappedFromPath, ILogger logger)
{
if (!Path.IsPathRooted(mappedToPath))
{
try
{
var fullPath = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(mappedFromPath)!, mappedToPath));
logger.LogDebug($"Found relative path in line mapping: '{mappedToPath}', interpreting it as '{fullPath}'");
mappedToPath = fullPath;
}
catch (Exception e)
{
logger.LogDebug($"Failed to compute absolute path for relative path in line mapping: '{mappedToPath}': {e}");
}
}
return mappedToPath;
}
}
}

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs
View File

@@ -63,7 +63,7 @@ namespace Semmle.Extraction.CSharp.Entities
{
if (method.MethodKind == MethodKind.ReducedExtension)
{
cx.Extractor.Logger.Log(Semmle.Util.Logging.Severity.Warning, "Reduced extension method symbols should not be directly extracted.");
cx.ExtractionContext.Logger.Log(Semmle.Util.Logging.Severity.Warning, "Reduced extension method symbols should not be directly extracted.");
}
return OrdinaryMethodFactory.Instance.CreateEntityFromSymbol(cx, method);

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/LineOrSpanDirective.cs
View File

@@ -28,7 +28,7 @@ namespace Semmle.Extraction.CSharp.Entities
var path = Symbol.File.ValueText;
if (!string.IsNullOrWhiteSpace(path))
{
path = NonGeneratedSourceLocation.TryAdjustRelativeMappedFilePath(path, Symbol.SyntaxTree.FilePath, Context.Extractor.Logger);
path = Context.TryAdjustRelativeMappedFilePath(path, Symbol.SyntaxTree.FilePath);
var file = File.Create(Context, path);
trapFile.directive_line_file(this, file);
}

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/PragmaChecksumDirective.cs
View File

@@ -12,7 +12,7 @@ namespace Semmle.Extraction.CSharp.Entities
protected override void PopulatePreprocessor(TextWriter trapFile)
{
var path = NonGeneratedSourceLocation.TryAdjustRelativeMappedFilePath(Symbol.File.ValueText, Symbol.SyntaxTree.FilePath, Context.Extractor.Logger);
var path = Context.TryAdjustRelativeMappedFilePath(Symbol.File.ValueText, Symbol.SyntaxTree.FilePath);
var file = File.Create(Context, path);
trapFile.pragma_checksums(this, file, Symbol.Guid.ToString(), Symbol.Bytes.ToString());
}

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs
View File

@@ -36,7 +36,7 @@ namespace Semmle.Extraction.CSharp.Entities
if (Symbol.TypeKind == TypeKind.Error)
{
UnknownType.Create(Context); // make sure this exists so we can use it in `TypeRef::getReferencedType`
Context.Extractor.MissingType(Symbol.ToString()!, Context.FromSource);
Context.ExtractionContext.MissingType(Symbol.ToString()!, Context.FromSource);
return;
}

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs
View File

@@ -36,7 +36,7 @@ namespace Semmle.Extraction.CSharp.Entities
}
else
{
Context.Extractor.MissingNamespace(name.ToFullString(), Context.FromSource);
Context.ExtractionContext.MissingNamespace(name.ToFullString(), Context.FromSource);
Context.ModelError(node, "Namespace not found");
return;
}

55
csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs
View File

@@ -10,6 +10,7 @@ using Microsoft.CodeAnalysis.CSharp;
using Semmle.Util;
using Semmle.Util.Logging;
using Semmle.Extraction.CSharp.Populators;
using System.Reflection;
namespace Semmle.Extraction.CSharp
{
@@ -18,7 +19,7 @@ namespace Semmle.Extraction.CSharp
/// </summary>
public class Analyser : IDisposable
{
protected Extraction.Extractor? extractor;
public ExtractionContext? ExtractionContext { get; protected set; }
protected CSharpCompilation? compilation;
protected CommonOptions? options;
private protected Entities.Compilation? compilationEntity;
@@ -38,14 +39,23 @@ namespace Semmle.Extraction.CSharp
public PathTransformer PathTransformer { get; }
protected Analyser(IProgressMonitor pm, ILogger logger, bool addAssemblyTrapPrefix, PathTransformer pathTransformer)
public IPathCache PathCache { get; }
protected Analyser(
IProgressMonitor pm,
ILogger logger,
PathTransformer pathTransformer,
IPathCache pathCache,
bool addAssemblyTrapPrefix)
{
Logger = logger;
PathTransformer = pathTransformer;
PathCache = pathCache;
this.addAssemblyTrapPrefix = addAssemblyTrapPrefix;
this.progressMonitor = pm;
Logger.Log(Severity.Info, "EXTRACTION STARTING at {0}", DateTime.Now);
stopWatch.Start();
progressMonitor = pm;
PathTransformer = pathTransformer;
}
/// <summary>
@@ -98,12 +108,12 @@ namespace Semmle.Extraction.CSharp
var def = reader.GetAssemblyDefinition();
assemblyIdentity = reader.GetString(def.Name) + " " + def.Version;
}
extractor.SetAssemblyFile(assemblyIdentity, refPath);
ExtractionContext.SetAssemblyFile(assemblyIdentity, refPath);
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
extractor.Message(new Message("Exception reading reference file", reference.FilePath, null, ex.StackTrace));
ExtractionContext.Message(new Message("Exception reading reference file", reference.FilePath, null, ex.StackTrace));
}
}
}
@@ -148,7 +158,7 @@ namespace Semmle.Extraction.CSharp
if (compilation.GetAssemblyOrModuleSymbol(r) is IAssemblySymbol assembly)
{
var cx = new Context(extractor, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
foreach (var module in assembly.Modules)
{
@@ -191,7 +201,7 @@ namespace Semmle.Extraction.CSharp
if (!upToDate)
{
var cx = new Context(extractor, compilation, trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
// Ensure that the file itself is populated in case the source file is totally empty
var root = tree.GetRoot();
Entities.File.Create(cx, root.SyntaxTree.FilePath);
@@ -213,7 +223,7 @@ namespace Semmle.Extraction.CSharp
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
extractor.Message(new Message($"Unhandled exception processing syntax tree. {ex.Message}", tree.FilePath, null, ex.StackTrace));
ExtractionContext.Message(new Message($"Unhandled exception processing syntax tree. {ex.Message}", tree.FilePath, null, ex.StackTrace));
}
}
@@ -221,7 +231,7 @@ namespace Semmle.Extraction.CSharp
{
try
{
var assemblyPath = extractor.OutputPath;
var assemblyPath = ExtractionContext.OutputPath;
var stopwatch = new Stopwatch();
stopwatch.Start();
var currentTaskId = IncrementTaskCount();
@@ -231,11 +241,11 @@ namespace Semmle.Extraction.CSharp
var assembly = compilation.Assembly;
var trapWriter = transformedAssemblyPath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
compilationTrapFile = trapWriter; // Dispose later
var cx = new Context(extractor, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
compilationEntity = Entities.Compilation.Create(cx);
extractor.CompilationInfos.ForEach(ci => trapWriter.Writer.compilation_info(compilationEntity, ci.key, ci.value));
ExtractionContext.CompilationInfos.ForEach(ci => trapWriter.Writer.compilation_info(compilationEntity, ci.key, ci.value));
ReportProgressTaskDone(currentTaskId, assemblyPath, trapWriter.TrapFile, stopwatch.Elapsed, AnalysisAction.Extracted);
}
@@ -318,7 +328,7 @@ namespace Semmle.Extraction.CSharp
/// <summary>
/// Number of errors encountered during extraction.
/// </summary>
private int ExtractorErrors => extractor?.Errors ?? 0;
private int ExtractorErrors => ExtractionContext?.Errors ?? 0;
/// <summary>
/// Number of errors encountered by the compiler.
@@ -333,11 +343,26 @@ namespace Semmle.Extraction.CSharp
/// <summary>
/// Logs information about the extractor.
/// </summary>
public void LogExtractorInfo(string extractorVersion)
public void LogExtractorInfo()
{
Logger.Log(Severity.Info, " Extractor: {0}", Environment.GetCommandLineArgs().First());
Logger.Log(Severity.Info, " Extractor version: {0}", extractorVersion);
Logger.Log(Severity.Info, " Extractor version: {0}", Version);
Logger.Log(Severity.Info, " Current working directory: {0}", Directory.GetCurrentDirectory());
}
private static string Version
{
get
{
// the attribute for the git information are always attached to the entry assembly by our build system
var assembly = Assembly.GetEntryAssembly();
var versionString = assembly?.GetCustomAttribute<AssemblyInformationalVersionAttribute>();
if (versionString == null)
{
return "unknown (not built from internal bazel workspace)";
}
return versionString.InformationalVersion;
}
}
}
}

25
csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using Microsoft.CodeAnalysis;
using Semmle.Extraction.Entities;
@@ -76,8 +77,8 @@ namespace Semmle.Extraction.CSharp
internal CommentProcessor CommentGenerator { get; } = new CommentProcessor();
public Context(Extraction.Extractor e, Compilation c, TrapWriter trapWriter, IExtractionScope scope, bool addAssemblyTrapPrefix)
: base(e, trapWriter, addAssemblyTrapPrefix)
public Context(ExtractionContext extractionContext, Compilation c, TrapWriter trapWriter, IExtractionScope scope, bool addAssemblyTrapPrefix)
: base(extractionContext, trapWriter, addAssemblyTrapPrefix)
{
Compilation = c;
this.scope = scope;
@@ -178,5 +179,25 @@ namespace Semmle.Extraction.CSharp
extractedGenerics.Add(entity.Label);
return true;
}
public string TryAdjustRelativeMappedFilePath(string mappedToPath, string mappedFromPath)
{
if (!Path.IsPathRooted(mappedToPath))
{
try
{
var fullPath = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(mappedFromPath)!, mappedToPath));
ExtractionContext.Logger.LogDebug($"Found relative path in line mapping: '{mappedToPath}', interpreting it as '{fullPath}'");
mappedToPath = fullPath;
}
catch (Exception e)
{
ExtractionContext.Logger.LogDebug($"Failed to compute absolute path for relative path in line mapping: '{mappedToPath}': {e}");
}
}
return mappedToPath;
}
}
}

33
csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs
View File

@@ -93,20 +93,13 @@ namespace Semmle.Extraction.CSharp
/// <returns><see cref="ExitCode"/></returns>
public static ExitCode Run(string[] args)
{
var stopwatch = new Stopwatch();
stopwatch.Start();
var analyzerStopwatch = new Stopwatch();
analyzerStopwatch.Start();
var options = Options.CreateWithEnvironment(args);
var workingDirectory = Directory.GetCurrentDirectory();
var compilerArgs = options.CompilerArguments.ToArray();
using var logger = MakeLogger(options.Verbosity, options.Console);
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
var pathTransformer = new PathTransformer(canonicalPathCache);
using var analyser = new TracingAnalyser(new LogProgressMonitor(logger), logger, options.AssemblySensitiveTrap, pathTransformer);
try
{
if (options.ProjectsToLoad.Any())
@@ -115,13 +108,20 @@ namespace Semmle.Extraction.CSharp
}
var compilerVersion = new CompilerVersion(options);
if (compilerVersion.SkipExtraction)
{
logger.Log(Severity.Warning, " Unrecognized compiler '{0}' because {1}", compilerVersion.SpecifiedCompiler, compilerVersion.SkipReason);
return ExitCode.Ok;
}
var workingDirectory = Directory.GetCurrentDirectory();
var compilerArgs = options.CompilerArguments.ToArray();
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
var pathTransformer = new PathTransformer(canonicalPathCache);
using var analyser = new TracingAnalyser(new LogProgressMonitor(logger), logger, pathTransformer, canonicalPathCache, options.AssemblySensitiveTrap);
var compilerArguments = CSharpCommandLineParser.Default.Parse(
compilerVersion.ArgsWithResponse,
workingDirectory,
@@ -144,7 +144,7 @@ namespace Semmle.Extraction.CSharp
return ExitCode.Ok;
}
return AnalyseTracing(workingDirectory, compilerArgs, analyser, compilerArguments, options, canonicalPathCache, stopwatch);
return AnalyseTracing(workingDirectory, compilerArgs, analyser, compilerArguments, options, analyzerStopwatch);
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
@@ -226,7 +226,7 @@ namespace Semmle.Extraction.CSharp
/// The resolved references will be added (thread-safely) to the supplied
/// list <paramref name="ret"/>.
/// </summary>
private static IEnumerable<Action> ResolveReferences(Microsoft.CodeAnalysis.CommandLineArguments args, Analyser analyser, CanonicalPathCache canonicalPathCache, BlockingCollection<MetadataReference> ret)
private static IEnumerable<Action> ResolveReferences(Microsoft.CodeAnalysis.CommandLineArguments args, Analyser analyser, BlockingCollection<MetadataReference> ret)
{
var referencePaths = new Lazy<string[]>(() => FixedReferencePaths(args).ToArray());
return args.MetadataReferences.Select<CommandLineReference, Action>(clref => () =>
@@ -235,7 +235,7 @@ namespace Semmle.Extraction.CSharp
{
if (File.Exists(clref.Reference))
{
var reference = MakeReference(clref, canonicalPathCache.GetCanonicalPath(clref.Reference));
var reference = MakeReference(clref, analyser.PathCache.GetCanonicalPath(clref.Reference));
ret.Add(reference);
}
else
@@ -252,7 +252,7 @@ namespace Semmle.Extraction.CSharp
var composed = referencePaths.Value
.Select(path => Path.Combine(path, clref.Reference))
.Where(path => File.Exists(path))
.Select(path => canonicalPathCache.GetCanonicalPath(path))
.Select(path => analyser.PathCache.GetCanonicalPath(path))
.FirstOrDefault();
if (composed is not null)
@@ -382,11 +382,10 @@ namespace Semmle.Extraction.CSharp
TracingAnalyser analyser,
CSharpCommandLineArguments compilerArguments,
Options options,
CanonicalPathCache canonicalPathCache,
Stopwatch stopwatch)
{
return Analyse(stopwatch, analyser, options,
references => ResolveReferences(compilerArguments, analyser, canonicalPathCache, references),
references => ResolveReferences(compilerArguments, analyser, references),
(analyser, syntaxTrees) =>
{
var paths = compilerArguments.SourceFiles
@@ -399,7 +398,7 @@ namespace Semmle.Extraction.CSharp
}
return ReadSyntaxTrees(
paths.Select(canonicalPathCache.GetCanonicalPath),
paths.Select(analyser.PathCache.GetCanonicalPath),
analyser,
compilerArguments.ParseOptions,
compilerArguments.Encoding,

18
csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs
View File

@@ -1,33 +1,25 @@
using System;
using System.Collections.Generic;
using System.IO;
using Microsoft.CodeAnalysis.CSharp;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp
{
public class StandaloneAnalyser : Analyser
{
public StandaloneAnalyser(IProgressMonitor pm, ILogger logger, bool addAssemblyTrapPrefix, PathTransformer pathTransformer)
: base(pm, logger, addAssemblyTrapPrefix, pathTransformer)
public StandaloneAnalyser(IProgressMonitor pm, ILogger logger, PathTransformer pathTransformer, IPathCache pathCache, bool addAssemblyTrapPrefix)
: base(pm, logger, pathTransformer, pathCache, addAssemblyTrapPrefix)
{
}
public void Initialize(string outputPath, IEnumerable<(string, string)> compilationInfos, CSharpCompilation compilationIn, CommonOptions options)
{
compilation = compilationIn;
extractor = new StandaloneExtractor(Directory.GetCurrentDirectory(), outputPath, compilationInfos, Logger, PathTransformer, options);
ExtractionContext = new ExtractionContext(Directory.GetCurrentDirectory(), [], outputPath, compilationInfos, Logger, PathTransformer, ExtractorMode.Standalone, options.QlTest);
this.options = options;
LogExtractorInfo(Extraction.Extractor.Version);
LogExtractorInfo();
SetReferencePaths();
}
#nullable disable warnings
public IEnumerable<string> MissingTypes => extractor.MissingTypes;
public IEnumerable<string> MissingNamespaces => extractor.MissingNamespaces;
#nullable restore warnings
}
}

48
csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs
View File

@@ -13,8 +13,8 @@ namespace Semmle.Extraction.CSharp
{
private bool init;
public TracingAnalyser(IProgressMonitor pm, ILogger logger, bool addAssemblyTrapPrefix, PathTransformer pathTransformer)
: base(pm, logger, addAssemblyTrapPrefix, pathTransformer)
public TracingAnalyser(IProgressMonitor pm, ILogger logger, PathTransformer pathTransformer, IPathCache pathCache, bool addAssemblyTrapPrefix)
: base(pm, logger, pathTransformer, pathCache, addAssemblyTrapPrefix)
{
}
@@ -25,7 +25,8 @@ namespace Semmle.Extraction.CSharp
/// <returns>A Boolean indicating whether to proceed with extraction.</returns>
public bool BeginInitialize(IEnumerable<string> roslynArgs)
{
return init = LogRoslynArgs(roslynArgs, Extraction.Extractor.Version);
LogExtractorInfo();
return init = LogRoslynArgs(roslynArgs);
}
/// <summary>
@@ -46,12 +47,12 @@ namespace Semmle.Extraction.CSharp
throw new InternalError("EndInitialize called without BeginInitialize returning true");
this.options = options;
this.compilation = compilation;
this.extractor = new TracingExtractor(cwd, args, GetOutputName(compilation, commandLineArguments), Logger, PathTransformer, options);
LogDiagnostics();
this.ExtractionContext = new ExtractionContext(cwd, args, GetOutputName(compilation, commandLineArguments), [], Logger, PathTransformer, ExtractorMode.None, options.QlTest);
var errorCount = LogDiagnostics();
SetReferencePaths();
CompilationErrors += FilteredDiagnostics.Count();
CompilationErrors += errorCount;
}
/// <summary>
@@ -59,9 +60,8 @@ namespace Semmle.Extraction.CSharp
/// </summary>
/// <param name="roslynArgs">The arguments passed to Roslyn.</param>
/// <returns>A Boolean indicating whether the same arguments have been logged previously.</returns>
private bool LogRoslynArgs(IEnumerable<string> roslynArgs, string extractorVersion)
private bool LogRoslynArgs(IEnumerable<string> roslynArgs)
{
LogExtractorInfo(extractorVersion);
Logger.Log(Severity.Info, $" Arguments to Roslyn: {string.Join(' ', roslynArgs)}");
var tempFile = Extractor.GetCSharpArgsLogPath(Path.GetRandomFileName());
@@ -137,27 +137,27 @@ namespace Semmle.Extraction.CSharp
return Path.Combine(commandLineArguments.OutputDirectory, commandLineArguments.OutputFileName);
}
#nullable disable warnings
/// <summary>
/// Logs detailed information about this invocation,
/// in the event that errors were detected.
/// </summary>
/// <returns>A Boolean indicating whether to proceed with extraction.</returns>
private void LogDiagnostics()
private int LogDiagnostics()
{
foreach (var error in FilteredDiagnostics)
var filteredDiagnostics = compilation!
.GetDiagnostics()
.Where(e => e.Severity >= DiagnosticSeverity.Error && !errorsToIgnore.Contains(e.Id))
.ToList();
foreach (var error in filteredDiagnostics)
{
Logger.Log(Severity.Error, " Compilation error: {0}", error);
}
if (FilteredDiagnostics.Any())
if (filteredDiagnostics.Count != 0)
{
foreach (var reference in compilation.References)
{
Logger.Log(Severity.Info, " Resolved reference {0}", reference.Display);
}
}
return filteredDiagnostics.Count;
}
private static readonly HashSet<string> errorsToIgnore = new HashSet<string>
@@ -166,17 +166,5 @@ namespace Semmle.Extraction.CSharp
"CS1589", // XML referencing not supported
"CS1569" // Error writing XML documentation
};
private IEnumerable<Diagnostic> FilteredDiagnostics
{
get
{
return extractor is null || extractor.Mode.HasFlag(ExtractorMode.Standalone) || compilation is null ? Enumerable.Empty<Diagnostic>() :
compilation.
GetDiagnostics().
Where(e => e.Severity >= DiagnosticSeverity.Error && !errorsToIgnore.Contains(e.Id));
}
}
#nullable restore warnings
}
}

14
csharp/extractor/Semmle.Extraction/Context.cs
View File

@@ -18,7 +18,7 @@ namespace Semmle.Extraction
/// <summary>
/// Access various extraction functions, e.g. logger, trap writer.
/// </summary>
public Extractor Extractor { get; }
public ExtractionContext ExtractionContext { get; }
/// <summary>
/// Access to the trap file.
@@ -51,7 +51,7 @@ namespace Semmle.Extraction
try
{
writingLabel = true;
entity.DefineLabel(TrapWriter.Writer, Extractor);
entity.DefineLabel(TrapWriter.Writer);
}
finally
{
@@ -190,9 +190,9 @@ namespace Semmle.Extraction
}
}
protected Context(Extractor extractor, TrapWriter trapWriter, bool shouldAddAssemblyTrapPrefix = false)
protected Context(ExtractionContext extractionContext, TrapWriter trapWriter, bool shouldAddAssemblyTrapPrefix = false)
{
Extractor = extractor;
ExtractionContext = extractionContext;
TrapWriter = trapWriter;
ShouldAddAssemblyTrapPrefix = shouldAddAssemblyTrapPrefix;
}
@@ -274,7 +274,7 @@ namespace Semmle.Extraction
bool duplicationGuard, deferred;
if (Extractor.Mode is ExtractorMode.Standalone)
if (ExtractionContext.Mode is ExtractorMode.Standalone)
{
duplicationGuard = false;
deferred = false;
@@ -398,7 +398,7 @@ namespace Semmle.Extraction
private void ExtractionError(Message msg)
{
new ExtractionMessage(this, msg);
Extractor.Message(msg);
ExtractionContext.Message(msg);
}
private void ExtractionError(InternalError error)
@@ -408,7 +408,7 @@ namespace Semmle.Extraction
private void ReportError(InternalError error)
{
if (!Extractor.Mode.HasFlag(ExtractorMode.Standalone))
if (!ExtractionContext.Mode.HasFlag(ExtractorMode.Standalone))
throw error;
ExtractionError(error);

4
csharp/extractor/Semmle.Extraction/Entities/Base/Entity.cs
View File

@@ -28,7 +28,7 @@ namespace Semmle.Extraction
public abstract TrapStackBehaviour TrapStackBehaviour { get; }
public void DefineLabel(TextWriter trapFile, Extractor extractor)
public void DefineLabel(TextWriter trapFile)
{
trapFile.WriteLabel(this);
trapFile.Write("=");
@@ -40,7 +40,7 @@ namespace Semmle.Extraction
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
trapFile.WriteLine("\"");
extractor.Message(new Message($"Unhandled exception generating id: {ex.Message}", ToString() ?? "", null, ex.StackTrace));
Context.ExtractionContext.Message(new Message($"Unhandled exception generating id: {ex.Message}", ToString() ?? "", null, ex.StackTrace));
}
trapFile.WriteLine();
}

2
csharp/extractor/Semmle.Extraction/Entities/Base/IEntity.cs
View File

@@ -48,6 +48,6 @@ namespace Semmle.Extraction
/// </summary>
TrapStackBehaviour TrapStackBehaviour { get; }
void DefineLabel(TextWriter trapFile, Extractor extractor);
void DefineLabel(TextWriter trapFile);
}
}

2
csharp/extractor/Semmle.Extraction/Entities/ExtractionMessage.cs
View File

@@ -25,7 +25,7 @@ namespace Semmle.Extraction.Entities
{
if (messageCount == limit + 1)
{
Context.Extractor.Logger.LogWarning($"Stopped logging extractor messages after reaching {limit}");
Context.ExtractionContext.Logger.LogWarning($"Stopped logging extractor messages after reaching {limit}");
}
return;
}

2
csharp/extractor/Semmle.Extraction/Entities/File.cs
View File

@@ -8,7 +8,7 @@ namespace Semmle.Extraction.Entities
: base(cx, path)
{
originalPath = path;
transformedPathLazy = new Lazy<PathTransformer.ITransformedPath>(() => Context.Extractor.PathTransformer.Transform(originalPath));
transformedPathLazy = new Lazy<PathTransformer.ITransformedPath>(() => Context.ExtractionContext.PathTransformer.Transform(originalPath));
}
protected readonly string originalPath;

31
csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs → csharp/extractor/Semmle.Extraction/Extractor/ExtractionContext.cs
View File

@@ -1,6 +1,4 @@
using System.Collections.Generic;
using System.Reflection;
using System.IO;
using Semmle.Util.Logging;
using CompilationInfo = (string key, string value);
@@ -9,20 +7,18 @@ namespace Semmle.Extraction
/// <summary>
/// Implementation of the main extractor state.
/// </summary>
public abstract class Extractor
public class ExtractionContext
{
public string Cwd { get; init; }
public string[] Args { get; init; }
public abstract ExtractorMode Mode { get; }
public ExtractorMode Mode { get; }
public string OutputPath { get; }
public IEnumerable<CompilationInfo> CompilationInfos { get; }
/// <summary>
/// Creates a new extractor instance for one compilation unit.
/// </summary>
/// <param name="logger">The object used for logging.</param>
/// <param name="pathTransformer">The object used for path transformations.</param>
protected Extractor(string cwd, string[] args, string outputPath, IEnumerable<CompilationInfo> compilationInfos, ILogger logger, PathTransformer pathTransformer)
public ExtractionContext(string cwd, string[] args, string outputPath, IEnumerable<CompilationInfo> compilationInfos, ILogger logger, PathTransformer pathTransformer, ExtractorMode mode, bool isQlTest)
{
OutputPath = outputPath;
Logger = logger;
@@ -30,6 +26,12 @@ namespace Semmle.Extraction
CompilationInfos = compilationInfos;
Cwd = cwd;
Args = args;
Mode = mode;
if (isQlTest)
{
Mode |= ExtractorMode.QlTest;
}
}
// Limit the number of error messages in the log file
@@ -107,21 +109,6 @@ namespace Semmle.Extraction
public ILogger Logger { get; private set; }
public static string Version
{
get
{
// the attribute for the git information are always attached to the entry assembly by our build system
var assembly = Assembly.GetEntryAssembly();
var versionString = assembly?.GetCustomAttribute<AssemblyInformationalVersionAttribute>();
if (versionString == null)
{
return "unknown (not built from internal bazel workspace)";
}
return versionString.InformationalVersion;
}
}
public PathTransformer PathTransformer { get; }
}
}

25
csharp/extractor/Semmle.Extraction/Extractor/StandaloneExtractor.cs
View File

@@ -1,25 +0,0 @@
using System.Collections.Generic;
using Semmle.Util.Logging;
namespace Semmle.Extraction
{
public class StandaloneExtractor : Extractor
{
public override ExtractorMode Mode { get; }
/// <summary>
/// Creates a new extractor instance for one compilation unit.
/// </summary>
/// <param name="logger">The object used for logging.</param>
/// <param name="pathTransformer">The object used for path transformations.</param>
public StandaloneExtractor(string cwd, string outputPath, IEnumerable<(string, string)> compilationInfos, ILogger logger, PathTransformer pathTransformer, CommonOptions options)
: base(cwd, [], outputPath, compilationInfos, logger, pathTransformer)
{
Mode = ExtractorMode.Standalone;
if (options.QlTest)
{
Mode |= ExtractorMode.QlTest;
}
}
}
}

25
csharp/extractor/Semmle.Extraction/Extractor/TracingExtractor.cs
View File

@@ -1,25 +0,0 @@
using Semmle.Util.Logging;
namespace Semmle.Extraction
{
public class TracingExtractor : Extractor
{
public override ExtractorMode Mode { get; }
/// <summary>
/// Creates a new extractor instance for one compilation unit.
/// </summary>
/// <param name="outputPath">The name of the output DLL/EXE, or null if not specified (standalone extraction).</param>
/// <param name="logger">The object used for logging.</param>
/// <param name="pathTransformer">The object used for path transformations.</param>
public TracingExtractor(string cwd, string[] args, string outputPath, ILogger logger, PathTransformer pathTransformer, CommonOptions options)
: base(cwd, args, outputPath, [], logger, pathTransformer)
{
Mode = ExtractorMode.None;
if (options.QlTest)
{
Mode |= ExtractorMode.QlTest;
}
}
}
}