hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-17 01:03:14 +01:00
Code Issues Packages Projects Releases Wiki Activity

C#: Add functionality to detect overlay mode and integrate in extraction context.

Browse Source
This commit is contained in:
Michael Nebel
2025-09-12 17:07:43 +02:00
parent 9026a5a82a
commit aa805580e3
8 changed files with 159 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs
View File

@@ -143,7 +143,8 @@ namespace Semmle.Extraction.CSharp.Standalone
var pathTransformer = new PathTransformer(canonicalPathCache);
var progressMonitor = new ExtractionProgress(logger);
using var analyser = new StandaloneAnalyser(progressMonitor, fileLogger, pathTransformer, canonicalPathCache, false);
var overlayInfo = OverlayInfoFactory.Make(logger, options.SrcDir);
using var analyser = new StandaloneAnalyser(progressMonitor, fileLogger, pathTransformer, canonicalPathCache, overlayInfo, false);
try
{
var extractionInput = new ExtractionInput(dependencyManager.AllSourceFiles, dependencyManager.ReferenceFiles, dependencyManager.CompilationInfos);

10
csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs
View File

@@ -41,16 +41,20 @@ namespace Semmle.Extraction.CSharp
public IPathCache PathCache { get; }
public IOverlayInfo OverlayInfo { get; }
protected Analyser(
IProgressMonitor pm,
ILogger logger,
PathTransformer pathTransformer,
IPathCache pathCache,
IOverlayInfo overlayInfo,
bool addAssemblyTrapPrefix)
{
Logger = logger;
PathTransformer = pathTransformer;
PathCache = pathCache;
OverlayInfo = overlayInfo;
this.addAssemblyTrapPrefix = addAssemblyTrapPrefix;
this.progressMonitor = pm;
@@ -158,7 +162,7 @@ namespace Semmle.Extraction.CSharp
if (compilation.GetAssemblyOrModuleSymbol(r) is IAssemblySymbol assembly)
{
var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), OverlayInfo, addAssemblyTrapPrefix);
foreach (var module in assembly.Modules)
{
@@ -195,7 +199,7 @@ namespace Semmle.Extraction.CSharp
var currentTaskId = IncrementTaskCount();
ReportProgressTaskStarted(currentTaskId, sourcePath);
var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), OverlayInfo, addAssemblyTrapPrefix);
// Ensure that the file itself is populated in case the source file is totally empty
var root = tree.GetRoot();
Entities.File.Create(cx, root.SyntaxTree.FilePath);
@@ -234,7 +238,7 @@ namespace Semmle.Extraction.CSharp
var assembly = compilation.Assembly;
var trapWriter = transformedAssemblyPath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
compilationTrapFile = trapWriter; // Dispose later
var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), OverlayInfo, addAssemblyTrapPrefix);
compilationEntity = Entities.Compilation.Create(cx);

2
csharp/extractor/Semmle.Extraction.CSharp/Extractor/BinaryLogAnalyser.cs
View File

@@ -8,7 +8,7 @@ namespace Semmle.Extraction.CSharp
public class BinaryLogAnalyser : Analyser
{
public BinaryLogAnalyser(IProgressMonitor pm, ILogger logger, PathTransformer pathTransformer, IPathCache pathCache, bool addAssemblyTrapPrefix)
: base(pm, logger, pathTransformer, pathCache, addAssemblyTrapPrefix)
: base(pm, logger, pathTransformer, pathCache, new TrivialOverlayInfo(), addAssemblyTrapPrefix)
{
}

9
csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs
View File

@@ -29,6 +29,12 @@ namespace Semmle.Extraction.CSharp
/// </summary>
public bool ShouldAddAssemblyTrapPrefix { get; }
/// <summary>
/// Holds if trap only should be created for types and member signatures (and not for expressions and statements).
/// This is the case for all unchanged files, when running in overlay mode.
/// </summary>
public bool OnlyScaffold { get; }
public IList<object> TrapStackSuffix { get; } = new List<object>();
private int GetNewId() => TrapWriter.IdCounter++;
@@ -523,13 +529,14 @@ namespace Semmle.Extraction.CSharp
internal CommentProcessor CommentGenerator { get; } = new CommentProcessor();
public Context(ExtractionContext extractionContext, Compilation c, TrapWriter trapWriter, IExtractionScope scope, bool shouldAddAssemblyTrapPrefix = false)
public Context(ExtractionContext extractionContext, Compilation c, TrapWriter trapWriter, IExtractionScope scope, IOverlayInfo overlayInfo, bool shouldAddAssemblyTrapPrefix = false)
{
ExtractionContext = extractionContext;
TrapWriter = trapWriter;
ShouldAddAssemblyTrapPrefix = shouldAddAssemblyTrapPrefix;
Compilation = c;
this.scope = scope;
OnlyScaffold = overlayInfo.IsOverlayMode && scope is SourceScope ss && overlayInfo.OnlyMakeScaffold(ss.SourceTree.FilePath);
}
public bool FromSource => scope is SourceScope;

124
csharp/extractor/Semmle.Extraction.CSharp/Extractor/OverlayInfo.cs Normal file
View File

@@ -0,0 +1,124 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp
{
public interface IOverlayInfo
{
/// <summary>
/// True, if the extractor is running in overlay mode.
/// </summary>
bool IsOverlayMode { get; }
/// <summary>
/// Returns true, if the given file is not in the set of changed files.
/// </summary>
/// <param name="filePath">A source file path</param>
bool OnlyMakeScaffold(string filePath);
}
/// <summary>
/// An instance of this class is used when overlay is not enabled.
/// </summary>
public class TrivialOverlayInfo : IOverlayInfo
{
public TrivialOverlayInfo() { }
public bool IsOverlayMode { get; } = false;
public bool OnlyMakeScaffold(string filePath) => false;
}
/// <summary>
/// An instance of this class is used for detecting
/// (1) Whether overlay is enabled.
/// (2) Fetch the changed files that should be fully extracted as a part
/// of the overlay extraction.
/// </summary>
public class OverlayInfo : IOverlayInfo
{
private readonly ILogger logger;
private readonly HashSet<string> changedFiles;
private readonly string srcDir;
public OverlayInfo(ILogger logger, string srcDir, string json)
{
this.logger = logger;
this.srcDir = srcDir;
changedFiles = ParseJson(json);
}
public bool IsOverlayMode { get; } = true;
public bool OnlyMakeScaffold(string filePath) => !changedFiles.Contains(filePath);
/// <summary>
/// Private type only used to parse overlay changes JSON files.
///
/// The content of such a file has the format
/// {
/// "changes": [
/// "app/controllers/about_controller.xyz",
/// "app/models/about.xyz"
/// ]
/// }
/// </summary>
public record ChangedFiles
{
public string[]? Changes { get; set; }
}
private HashSet<string> ParseJson(string json)
{
try
{
var options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true };
var obj = JsonSerializer.Deserialize<ChangedFiles>(json, options);
return obj?.Changes is string[] changes
? changes.Select(change => Path.Join(srcDir, change)).ToHashSet()
: [];
}
catch (JsonException)
{
logger.LogError("Overlay: Unable to parse the JSON content from the overlay changes file.");
return [];
}
}
}
public static class OverlayInfoFactory
{
/// <summary>
/// The returned object is used to decide, whether
/// (1) The extractor is running in overlay mode.
/// (2) Which files to only extract scaffolds for (unchanged files)
/// </summary>
/// <param name="logger">A logger</param>
/// <param name="srcDir">The (overlay) source directory</param>
/// <returns>An overlay information object.</returns>
public static IOverlayInfo Make(ILogger logger, string srcDir)
{
if (EnvironmentVariables.GetOverlayChangesFilePath() is string path)
{
logger.LogInfo($"Overlay: Reading overlay changes from file '{path}'.");
try
{
var json = File.ReadAllText(path);
return new OverlayInfo(logger, srcDir, json);
}
catch
{
logger.LogError("Overlay: Unexpected error while reading the overlay changes file.");
}
}
logger.LogInfo("Overlay: Overlay mode not enabled.");
return new TrivialOverlayInfo();
}
}
}

4
csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs
View File

@@ -8,8 +8,8 @@ namespace Semmle.Extraction.CSharp
{
public class StandaloneAnalyser : Analyser
{
public StandaloneAnalyser(IProgressMonitor pm, ILogger logger, PathTransformer pathTransformer, IPathCache pathCache, bool addAssemblyTrapPrefix)
: base(pm, logger, pathTransformer, pathCache, addAssemblyTrapPrefix)
public StandaloneAnalyser(IProgressMonitor pm, ILogger logger, PathTransformer pathTransformer, IPathCache pathCache, IOverlayInfo overlayInfo, bool addAssemblyTrapPrefix)
: base(pm, logger, pathTransformer, pathCache, overlayInfo, addAssemblyTrapPrefix)
{
}

2
csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs
View File

@@ -14,7 +14,7 @@ namespace Semmle.Extraction.CSharp
private bool init;
public TracingAnalyser(IProgressMonitor pm, ILogger logger, PathTransformer pathTransformer, IPathCache pathCache, bool addAssemblyTrapPrefix)
: base(pm, logger, pathTransformer, pathCache, addAssemblyTrapPrefix)
: base(pm, logger, pathTransformer, pathCache, new TrivialOverlayInfo(), addAssemblyTrapPrefix)
{
}

14
csharp/extractor/Semmle.Util/EnvironmentVariables.cs
View File

@@ -53,5 +53,19 @@ namespace Semmle.Util
{
return Environment.GetEnvironmentVariable(name)?.Split(" ", StringSplitOptions.RemoveEmptyEntries) ?? [];
}
/// <summary>
/// Used to
/// (1) Detect whether the extractor should run in overlay mode.
/// (2) Returns the path to the file containing a list of changed files
/// in JSON format.
///
/// The environment variable is only set in case the extraction is supposed to be
/// performed in overlay mode. Furthermore, this only applies to buildless extraction.
/// </summary>
public static string? GetOverlayChangesFilePath()
{
return Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OVERLAY_CHANGES");
}
}
}