hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-17 01:03:14 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #20425 from michaelnebel/csharp/basicextractoroverlay

Browse Source 
C#: Overlay extraction support.
This commit is contained in:
Michael Nebel
2025-11-12 15:25:57 +01:00
committed by GitHub
parent 54ff206fde 0541dccc78
commit 085d3c85cd
75 changed files with 13343 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs
View File

@@ -2,6 +2,7 @@ using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
@@ -53,6 +54,20 @@ namespace Semmle.Extraction.CSharp.Standalone
}
progressMonitor.MissingSummary(analyser.ExtractionContext!.MissingTypes.Count(), analyser.ExtractionContext!.MissingNamespaces.Count());
// If extracting a base database, we need to create an empty metadata file.
if (EnvironmentVariables.GetBaseMetaDataOutPath() is string baseMetaDataOutPath)
{
try
{
analyser.Logger.LogInfo($"Creating base metadata file at {baseMetaDataOutPath}");
File.WriteAllText(baseMetaDataOutPath, string.Empty);
}
catch (Exception ex)
{
analyser.Logger.LogError($"Failed to create base metadata file: {ex.Message}");
}
}
});
}
finally
@@ -143,7 +158,8 @@ namespace Semmle.Extraction.CSharp.Standalone
var pathTransformer = new PathTransformer(canonicalPathCache);
var progressMonitor = new ExtractionProgress(logger);
using var analyser = new StandaloneAnalyser(progressMonitor, fileLogger, pathTransformer, canonicalPathCache, false);
var overlayInfo = OverlayInfoFactory.Make(logger, options.SrcDir);
using var analyser = new StandaloneAnalyser(progressMonitor, fileLogger, pathTransformer, canonicalPathCache, overlayInfo, false);
try
{
var extractionInput = new ExtractionInput(dependencyManager.AllSourceFiles, dependencyManager.ReferenceFiles, dependencyManager.CompilationInfos);

13
csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs
View File

@@ -57,8 +57,21 @@ namespace Semmle.Extraction.CSharp.Entities
public override void Populate(TextWriter trapFile)
{
// In this case, we don't extract the attribute again, as it was extracted using * ID
// originally and we re-use that.
if (Context.OnlyScaffold && (ReportingLocation is null || !ReportingLocation.IsInSource))
{
return;
}
var type = Type.Create(Context, Symbol.AttributeClass);
trapFile.attributes(this, kind, type.TypeRef, entity);
if (Context.OnlyScaffold)
{
return;
}
WriteLocationToTrap(trapFile.attribute_location, this, Location);
if (attributeSyntax is not null)

10
csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentBlock.cs
View File

@@ -10,9 +10,13 @@ namespace Semmle.Extraction.CSharp.Entities
public override void Populate(TextWriter trapFile)
{
if (Context.OnlyScaffold)
{
return;
}
trapFile.commentblock(this);
WriteLocationToTrap(trapFile.commentblock_location, this, Context.CreateLocation(Symbol.Location));
Symbol.CommentLines.ForEach((l, child) => trapFile.commentblock_child(this, l, child));
WriteLocationToTrap(trapFile.commentblock_location, this, Context.CreateLocation(Symbol.Location));
}
public override bool NeedsPopulation => true;
@@ -27,6 +31,10 @@ namespace Semmle.Extraction.CSharp.Entities
public void BindTo(Label entity, CommentBinding binding)
{
if (Context.OnlyScaffold)
{
return;
}
Context.TrapWriter.Writer.commentblock_binding(this, entity, binding);
}

7
csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs
View File

@@ -21,9 +21,14 @@ namespace Semmle.Extraction.CSharp.Entities
public override void Populate(TextWriter trapFile)
{
location = Context.CreateLocation(Location);
if (Context.OnlyScaffold)
{
return;
}
trapFile.commentline(this, Type == CommentLineType.MultilineContinuation ? CommentLineType.Multiline : Type, Text, RawText);
location = Context.CreateLocation(Location);
WriteLocationToTrap(trapFile.commentline_location, this, location);
}
public override Microsoft.CodeAnalysis.Location? ReportingLocation => location?.Symbol;

5
csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/CompilerDiagnostic.cs
View File

@@ -21,6 +21,11 @@ namespace Semmle.Extraction.CSharp.Entities
protected override void Populate(TextWriter trapFile)
{
if (Context.OnlyScaffold)
{
return;
}
var key = diagnostic.Id;
var messageCount = compilation.messageCounts.AddOrUpdate(key, 1, (_, c) => c + 1);
if (messageCount > limit)

24
csharp/extractor/Semmle.Extraction.CSharp/Entities/Constructor.cs
View File

@@ -29,9 +29,17 @@ namespace Semmle.Extraction.CSharp.Entities
ContainingType!.PopulateGenerics();
trapFile.constructors(this, Symbol.ContainingType.Name, ContainingType, (Constructor)OriginalDefinition);
if (Context.ExtractLocation(Symbol) && (!IsDefault || IsBestSourceLocation))
if (Symbol.IsImplicitlyDeclared)
{
WriteLocationToTrap(trapFile.constructor_location, this, Location);
var lineCounts = new LineCounts() { Total = 2, Code = 1, Comment = 0 };
trapFile.numlines(this, lineCounts);
}
ExtractCompilerGenerated(trapFile);
if (Context.OnlyScaffold)
{
return;
}
if (MakeSynthetic)
@@ -40,12 +48,11 @@ namespace Semmle.Extraction.CSharp.Entities
Statements.SyntheticEmptyBlock.Create(Context, this, 0, Location);
}
if (Symbol.IsImplicitlyDeclared)
if (Context.ExtractLocation(Symbol) && (!IsDefault || IsBestSourceLocation))
{
var lineCounts = new LineCounts() { Total = 2, Code = 1, Comment = 0 };
trapFile.numlines(this, lineCounts);
WriteLocationToTrap(trapFile.constructor_location, this, Location);
}
ExtractCompilerGenerated(trapFile);
}
protected override void ExtractInitializers(TextWriter trapFile)
@@ -53,7 +60,7 @@ namespace Semmle.Extraction.CSharp.Entities
// Do not extract initializers for constructed types.
// Extract initializers for constructors with a body, primary constructors
// and default constructors for classes and structs declared in source code.
if (Block is null && ExpressionBody is null && !MakeSynthetic)
if (Block is null && ExpressionBody is null && !MakeSynthetic || Context.OnlyScaffold)
{
return;
}
@@ -106,6 +113,7 @@ namespace Semmle.Extraction.CSharp.Entities
}
var baseConstructorTarget = Create(Context, baseConstructor);
var info = new ExpressionInfo(Context,
AnnotatedTypeSymbol.CreateNotAnnotated(baseType),
Location,
@@ -179,7 +187,7 @@ namespace Semmle.Extraction.CSharp.Entities
/// </summary>
private bool IsBestSourceLocation => ReportingLocation is not null && Context.IsLocationInContext(ReportingLocation);
private bool MakeSynthetic => IsPrimary || (IsDefault && IsBestSourceLocation);
private bool MakeSynthetic => (IsPrimary || (IsDefault && IsBestSourceLocation)) && !Context.OnlyScaffold;
[return: NotNullIfNotNull(nameof(constructor))]
public static new Constructor? Create(Context cx, IMethodSymbol? constructor)

1
csharp/extractor/Semmle.Extraction.CSharp/Entities/Destructor.cs
View File

@@ -15,6 +15,7 @@ namespace Semmle.Extraction.CSharp.Entities
ContainingType!.PopulateGenerics();
trapFile.destructors(this, $"~{Symbol.ContainingType.Name}", ContainingType, OriginalDefinition(Context, this, Symbol));
if (Context.ExtractLocation(Symbol))
{
WriteLocationToTrap(trapFile.destructor_location, this, Location);

8
csharp/extractor/Semmle.Extraction.CSharp/Entities/Event.cs
View File

@@ -37,7 +37,6 @@ namespace Semmle.Extraction.CSharp.Entities
Method.Create(Context, remover);
PopulateModifiers(trapFile);
BindComments();
var declSyntaxReferences = IsSourceDeclaration
? Symbol.DeclaringSyntaxReferences.Select(d => d.GetSyntax()).ToArray()
@@ -51,6 +50,13 @@ namespace Semmle.Extraction.CSharp.Entities
TypeMention.Create(Context, syntax.ExplicitInterfaceSpecifier!.Name, this, explicitInterface);
}
if (Context.OnlyScaffold)
{
return;
}
BindComments();
if (Context.ExtractLocation(Symbol))
{
WriteLocationsToTrap(trapFile.event_location, this, Locations);

5
csharp/extractor/Semmle.Extraction.CSharp/Entities/ExtractionMessage.cs
View File

@@ -28,6 +28,11 @@ namespace Semmle.Extraction.CSharp.Entities
protected override void Populate(TextWriter trapFile)
{
if (Context.OnlyScaffold)
{
return;
}
// For the time being we're counting the number of messages per severity, we could introduce other groupings in the future
var key = msg.Severity.ToString();
groupedMessageCounts.AddOrUpdate(key, 1, (_, c) => c + 1);

5
csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs
View File

@@ -49,6 +49,11 @@ namespace Semmle.Extraction.CSharp.Entities
}
}
if (Context.OnlyScaffold)
{
return;
}
if (Context.ExtractLocation(Symbol))
{
WriteLocationsToTrap(trapFile.field_location, this, Locations);

38
csharp/extractor/Semmle.Extraction.CSharp/Entities/Indexer.cs
View File

@@ -19,10 +19,6 @@ namespace Semmle.Extraction.CSharp.Entities
var type = Type.Create(Context, Symbol.Type);
trapFile.indexers(this, Symbol.GetName(useMetadataName: true), ContainingType!, type.TypeRef, OriginalDefinition);
if (Context.ExtractLocation(Symbol))
{
WriteLocationsToTrap(trapFile.indexer_location, this, Locations);
}
var getter = BodyDeclaringSymbol.GetMethod;
var setter = BodyDeclaringSymbol.SetMethod;
@@ -42,20 +38,8 @@ namespace Semmle.Extraction.CSharp.Entities
Parameter.Create(Context, Symbol.Parameters[i], this, original);
}
if (IsSourceDeclaration)
{
var expressionBody = ExpressionBody;
if (expressionBody is not null)
{
// The expression may need to reference parameters in the getter.
// So we need to arrange that the expression is populated after the getter.
Context.PopulateLater(() => Expression.CreateFromNode(new ExpressionNodeInfo(Context, expressionBody, this, 0).SetType(Symbol.GetAnnotatedType())));
}
}
PopulateAttributes();
PopulateModifiers(trapFile);
BindComments();
var declSyntaxReferences = IsSourceDeclaration
? Symbol.DeclaringSyntaxReferences.
@@ -70,6 +54,28 @@ namespace Semmle.Extraction.CSharp.Entities
TypeMention.Create(Context, syntax.ExplicitInterfaceSpecifier!.Name, this, explicitInterface);
}
if (Context.OnlyScaffold)
{
return;
}
if (Context.ExtractLocation(Symbol))
{
WriteLocationsToTrap(trapFile.indexer_location, this, Locations);
}
if (IsSourceDeclaration)
{
var expressionBody = ExpressionBody;
if (expressionBody is not null)
{
// The expression may need to reference parameters in the getter.
// So we need to arrange that the expression is populated after the getter.
Context.PopulateLater(() => Expression.CreateFromNode(new ExpressionNodeInfo(Context, expressionBody, this, 0).SetType(Symbol.GetAnnotatedType())));
}
}
BindComments();
foreach (var syntax in declSyntaxReferences)
TypeMention.Create(Context, syntax.Type, this, type);

5
csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalVariable.cs
View File

@@ -41,6 +41,11 @@ namespace Semmle.Extraction.CSharp.Entities
trapFile.localvars(this, Kinds.VariableKind.None, Symbol.Name, @var, Type.Create(Context, parent.Type).TypeRef, parent);
}
if (Context.OnlyScaffold)
{
return;
}
WriteLocationToTrap(trapFile.localvar_location, this, Location);
DefineConstantValue(trapFile);

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/Method.cs
View File

@@ -48,7 +48,7 @@ namespace Semmle.Extraction.CSharp.Entities
protected virtual void PopulateMethodBody(TextWriter trapFile)
{
if (!IsSourceDeclaration)
if (!IsSourceDeclaration || Context.OnlyScaffold)
return;
var block = Block;

7
csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs
View File

@@ -35,7 +35,6 @@ namespace Semmle.Extraction.CSharp.Entities
var ns = Namespace.Create(Context, @namespace);
trapFile.namespace_declarations(this, ns);
WriteLocationToTrap(trapFile.namespace_declaration_location, this, Context.CreateLocation(node.Name.GetLocation()));
var visitor = new Populators.TypeOrNamespaceVisitor(Context, trapFile, this);
@@ -48,6 +47,12 @@ namespace Semmle.Extraction.CSharp.Entities
{
trapFile.parent_namespace_declaration(this, parent);
}
if (Context.OnlyScaffold)
{
return;
}
WriteLocationToTrap(trapFile.namespace_declaration_location, this, Context.CreateLocation(node.Name.GetLocation()));
}
public static NamespaceDeclaration Create(Context cx, BaseNamespaceDeclarationSyntax decl, NamespaceDeclaration parent)

15
csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs
View File

@@ -34,6 +34,16 @@ namespace Semmle.Extraction.CSharp.Entities
var returnType = Type.Create(Context, Symbol.ReturnType);
trapFile.methods(this, Name, ContainingType, returnType.TypeRef, OriginalDefinition);
PopulateGenerics(trapFile);
Overrides(trapFile);
ExtractRefReturn(trapFile, Symbol, this);
ExtractCompilerGenerated(trapFile);
if (Context.OnlyScaffold)
{
return;
}
if (IsSourceDeclaration)
{
foreach (var declaration in Symbol.DeclaringSyntaxReferences.Select(s => s.GetSyntax()).OfType<MethodDeclarationSyntax>())
@@ -47,11 +57,6 @@ namespace Semmle.Extraction.CSharp.Entities
{
WriteLocationsToTrap(trapFile.method_location, this, Locations);
}
PopulateGenerics(trapFile);
Overrides(trapFile);
ExtractRefReturn(trapFile, Symbol, this);
ExtractCompilerGenerated(trapFile);
}
private bool IsCompilerGeneratedDelegate() =>

5
csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs
View File

@@ -115,6 +115,11 @@ namespace Semmle.Extraction.CSharp.Entities
var type = Type.Create(Context, Symbol.Type);
trapFile.@params(this, Name, type.TypeRef, Ordinal, ParamKind, Parent!, Original);
if (Context.OnlyScaffold)
{
return;
}
if (Context.ExtractLocation(Symbol))
{
var locations = Context.GetLocations(Symbol);

7
csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/PreprocessorDirective.cs
View File

@@ -13,10 +13,15 @@ namespace Semmle.Extraction.CSharp.Entities
PopulatePreprocessor(trapFile);
trapFile.preprocessor_directive_active(this, Symbol.IsActive);
WriteLocationToTrap(trapFile.preprocessor_directive_location, this, Context.CreateLocation(ReportingLocation));
var compilation = Compilation.Create(Context);
trapFile.preprocessor_directive_compilation(this, compilation);
if (Context.OnlyScaffold)
{
return;
}
WriteLocationToTrap(trapFile.preprocessor_directive_location, this, Context.CreateLocation(ReportingLocation));
}
protected abstract void PopulatePreprocessor(TextWriter trapFile);

8
csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs
View File

@@ -40,7 +40,6 @@ namespace Semmle.Extraction.CSharp.Entities
{
PopulateAttributes();
PopulateModifiers(trapFile);
BindComments();
PopulateNullability(trapFile, Symbol.GetAnnotatedType());
PopulateRefKind(trapFile, Symbol.RefKind);
@@ -69,6 +68,13 @@ namespace Semmle.Extraction.CSharp.Entities
TypeMention.Create(Context, syntax.ExplicitInterfaceSpecifier!.Name, this, explicitInterface);
}
if (Context.OnlyScaffold)
{
return;
}
BindComments();
if (Context.ExtractLocation(Symbol))
{
WriteLocationsToTrap(trapFile.property_location, this, Locations);

5
csharp/extractor/Semmle.Extraction.CSharp/Entities/TypeMention.cs
View File

@@ -59,6 +59,11 @@ namespace Semmle.Extraction.CSharp.Entities
protected override void Populate(TextWriter trapFile)
{
if (Context.OnlyScaffold)
{
return;
}
switch (syntax.Kind())
{
case SyntaxKind.ArrayType:

6
csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/DynamicType.cs
View File

@@ -16,10 +16,14 @@ namespace Semmle.Extraction.CSharp.Entities
public override void Populate(TextWriter trapFile)
{
trapFile.types(this, Kinds.TypeKind.DYNAMIC, "dynamic");
WriteLocationToTrap(trapFile.type_location, this, Location);
trapFile.has_modifiers(this, Modifier.Create(Context, "public"));
trapFile.parent_namespace(this, Namespace.Create(Context, Context.Compilation.GlobalNamespace));
if (Context.OnlyScaffold)
{
return;
}
WriteLocationToTrap(trapFile.type_location, this, Location);
}
public override void WriteId(EscapingTextWriter trapFile)

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs
View File

@@ -81,7 +81,7 @@ namespace Semmle.Extraction.CSharp.Entities
}
// Class location
if (!Symbol.IsGenericType || Symbol.IsReallyUnbound())
if ((!Symbol.IsGenericType || Symbol.IsReallyUnbound()) && !Context.OnlyScaffold)
{
WriteLocationsToTrap(trapFile.type_location, this, Locations);
}

4
csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TupleType.cs
View File

@@ -51,6 +51,10 @@ namespace Semmle.Extraction.CSharp.Entities
trapFile.tuple_element(this, index++, element);
}
if (Context.OnlyScaffold)
{
return;
}
// Note: symbol.Locations seems to be very inconsistent
// about what locations are available for a tuple type.
// Sometimes it's the source code, and sometimes it's empty.

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs
View File

@@ -222,7 +222,7 @@ namespace Semmle.Extraction.CSharp.Entities
private IEnumerable<BaseTypeSyntax> GetBaseTypeDeclarations()
{
if (!IsSourceDeclaration || !Symbol.FromSource())
if (!IsSourceDeclaration || !Symbol.FromSource() || Context.OnlyScaffold)
{
return Enumerable.Empty<BaseTypeSyntax>();
}

5
csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameter.cs
View File

@@ -26,6 +26,11 @@ namespace Semmle.Extraction.CSharp.Entities
var parentNs = Namespace.Create(Context, Symbol.TypeParameterKind == TypeParameterKind.Method ? Context.Compilation.GlobalNamespace : Symbol.ContainingNamespace);
trapFile.parent_namespace(this, parentNs);
if (Context.OnlyScaffold)
{
return;
}
if (Context.ExtractLocation(Symbol))
{
var locations = Context.GetLocations(Symbol);

11
csharp/extractor/Semmle.Extraction.CSharp/Entities/UserOperator.cs
View File

@@ -26,6 +26,14 @@ namespace Semmle.Extraction.CSharp.Entities
returnType.TypeRef,
(UserOperator)OriginalDefinition);
ContainingType.PopulateGenerics();
Overrides(trapFile);
if (Context.OnlyScaffold)
{
return;
}
if (Context.ExtractLocation(Symbol))
{
WriteLocationsToTrap(trapFile.operator_location, this, Locations);
@@ -39,9 +47,6 @@ namespace Semmle.Extraction.CSharp.Entities
foreach (var declaration in declSyntaxReferences.OfType<ConversionOperatorDeclarationSyntax>())
TypeMention.Create(Context, declaration.Type, this, returnType);
}
ContainingType.PopulateGenerics();
Overrides(trapFile);
}
public override bool NeedsPopulation => Context.Defines(Symbol) || IsImplicitOperator(out _);

5
csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs
View File

@@ -20,6 +20,11 @@ namespace Semmle.Extraction.CSharp.Entities
protected override void Populate(TextWriter trapFile)
{
if (Context.OnlyScaffold)
{
return;
}
// This is guaranteed to be non-null as we only deal with "using namespace" not "using X = Y"
var name = node.Name!;

10
csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs
View File

@@ -41,16 +41,20 @@ namespace Semmle.Extraction.CSharp
public IPathCache PathCache { get; }
public IOverlayInfo OverlayInfo { get; }
protected Analyser(
IProgressMonitor pm,
ILogger logger,
PathTransformer pathTransformer,
IPathCache pathCache,
IOverlayInfo overlayInfo,
bool addAssemblyTrapPrefix)
{
Logger = logger;
PathTransformer = pathTransformer;
PathCache = pathCache;
OverlayInfo = overlayInfo;
this.addAssemblyTrapPrefix = addAssemblyTrapPrefix;
this.progressMonitor = pm;
@@ -158,7 +162,7 @@ namespace Semmle.Extraction.CSharp
if (compilation.GetAssemblyOrModuleSymbol(r) is IAssemblySymbol assembly)
{
var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), OverlayInfo, addAssemblyTrapPrefix);
foreach (var module in assembly.Modules)
{
@@ -195,7 +199,7 @@ namespace Semmle.Extraction.CSharp
var currentTaskId = IncrementTaskCount();
ReportProgressTaskStarted(currentTaskId, sourcePath);
var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), OverlayInfo, addAssemblyTrapPrefix);
// Ensure that the file itself is populated in case the source file is totally empty
var root = tree.GetRoot();
Entities.File.Create(cx, root.SyntaxTree.FilePath);
@@ -234,7 +238,7 @@ namespace Semmle.Extraction.CSharp
var assembly = compilation.Assembly;
var trapWriter = transformedAssemblyPath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
compilationTrapFile = trapWriter; // Dispose later
var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), OverlayInfo, addAssemblyTrapPrefix);
compilationEntity = Entities.Compilation.Create(cx);

2
csharp/extractor/Semmle.Extraction.CSharp/Extractor/BinaryLogAnalyser.cs
View File

@@ -8,7 +8,7 @@ namespace Semmle.Extraction.CSharp
public class BinaryLogAnalyser : Analyser
{
public BinaryLogAnalyser(IProgressMonitor pm, ILogger logger, PathTransformer pathTransformer, IPathCache pathCache, bool addAssemblyTrapPrefix)
: base(pm, logger, pathTransformer, pathCache, addAssemblyTrapPrefix)
: base(pm, logger, pathTransformer, pathCache, new TrivialOverlayInfo(), addAssemblyTrapPrefix)
{
}

18
csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs
View File

@@ -29,6 +29,12 @@ namespace Semmle.Extraction.CSharp
/// </summary>
public bool ShouldAddAssemblyTrapPrefix { get; }
/// <summary>
/// Holds if trap only should be created for types and member signatures (and not for expressions and statements).
/// This is the case for all unchanged files, when running in overlay mode.
/// </summary>
public bool OnlyScaffold { get; }
public IList<object> TrapStackSuffix { get; } = new List<object>();
private int GetNewId() => TrapWriter.IdCounter++;
@@ -523,13 +529,16 @@ namespace Semmle.Extraction.CSharp
internal CommentProcessor CommentGenerator { get; } = new CommentProcessor();
public Context(ExtractionContext extractionContext, Compilation c, TrapWriter trapWriter, IExtractionScope scope, bool shouldAddAssemblyTrapPrefix = false)
public Context(ExtractionContext extractionContext, Compilation c, TrapWriter trapWriter, IExtractionScope scope, IOverlayInfo overlayInfo, bool shouldAddAssemblyTrapPrefix = false)
{
ExtractionContext = extractionContext;
TrapWriter = trapWriter;
ShouldAddAssemblyTrapPrefix = shouldAddAssemblyTrapPrefix;
Compilation = c;
this.scope = scope;
OnlyScaffold = overlayInfo.IsOverlayMode && (
IsAssemblyScope
|| (scope is SourceScope ss && overlayInfo.OnlyMakeScaffold(ss.SourceTree.FilePath)));
}
public bool FromSource => scope is SourceScope;
@@ -552,7 +561,8 @@ namespace Semmle.Extraction.CSharp
public bool ExtractLocation(ISymbol symbol) =>
SymbolEqualityComparer.Default.Equals(symbol, symbol.OriginalDefinition) &&
scope.InScope(symbol);
scope.InScope(symbol) &&
!OnlyScaffold;
/// <summary>
/// Gets the locations of the symbol that are either
@@ -621,6 +631,10 @@ namespace Semmle.Extraction.CSharp
/// <param name="l">Location of the entity.</param>
public void BindComments(Entity entity, Microsoft.CodeAnalysis.Location? l)
{
if (OnlyScaffold)
{
return;
}
var duplicationGuardKey = GetCurrentTagStackKey();
CommentGenerator.AddElement(entity.Label, duplicationGuardKey, l);
}

124
csharp/extractor/Semmle.Extraction.CSharp/Extractor/OverlayInfo.cs Normal file
View File

@@ -0,0 +1,124 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp
{
public interface IOverlayInfo
{
/// <summary>
/// True, if the extractor is running in overlay mode.
/// </summary>
bool IsOverlayMode { get; }
/// <summary>
/// Returns true, if the given file is not in the set of changed files.
/// </summary>
/// <param name="filePath">A source file path</param>
bool OnlyMakeScaffold(string filePath);
}
/// <summary>
/// An instance of this class is used when overlay is not enabled.
/// </summary>
public class TrivialOverlayInfo : IOverlayInfo
{
public TrivialOverlayInfo() { }
public bool IsOverlayMode { get; } = false;
public bool OnlyMakeScaffold(string filePath) => false;
}
/// <summary>
/// An instance of this class is used for detecting
/// (1) Whether overlay is enabled.
/// (2) Fetch the changed files that should be fully extracted as a part
/// of the overlay extraction.
/// </summary>
public class OverlayInfo : IOverlayInfo
{
private readonly ILogger logger;
private readonly HashSet<string> changedFiles;
private readonly string srcDir;
public OverlayInfo(ILogger logger, string srcDir, string json)
{
this.logger = logger;
this.srcDir = srcDir;
changedFiles = ParseJson(json);
}
public bool IsOverlayMode { get; } = true;
public bool OnlyMakeScaffold(string filePath) => !changedFiles.Contains(filePath);
/// <summary>
/// Private type only used to parse overlay changes JSON files.
///
/// The content of such a file has the format
/// {
/// "changes": [
/// "app/controllers/about_controller.xyz",
/// "app/models/about.xyz"
/// ]
/// }
/// </summary>
private record ChangedFiles
{
public string[]? Changes { get; set; }
}
private HashSet<string> ParseJson(string json)
{
try
{
var options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true };
var obj = JsonSerializer.Deserialize<ChangedFiles>(json, options);
return obj?.Changes is string[] changes
? changes.Select(change => Path.Join(srcDir, change)).ToHashSet()
: [];
}
catch (JsonException)
{
logger.LogError("Overlay: Unable to parse the JSON content from the overlay changes file.");
return [];
}
}
}
public static class OverlayInfoFactory
{
/// <summary>
/// The returned object is used to decide, whether
/// (1) The extractor is running in overlay mode.
/// (2) Which files to only extract scaffolds for (unchanged files)
/// </summary>
/// <param name="logger">A logger</param>
/// <param name="srcDir">The (overlay) source directory</param>
/// <returns>An overlay information object.</returns>
public static IOverlayInfo Make(ILogger logger, string srcDir)
{
if (EnvironmentVariables.GetOverlayChangesFilePath() is string path)
{
logger.LogInfo($"Overlay: Reading overlay changes from file '{path}'.");
try
{
var json = File.ReadAllText(path);
return new OverlayInfo(logger, srcDir, json);
}
catch
{
logger.LogError("Overlay: Unexpected error while reading the overlay changes file.");
}
}
logger.LogInfo("Overlay: Overlay mode not enabled.");
return new TrivialOverlayInfo();
}
}
}

4
csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs
View File

@@ -8,8 +8,8 @@ namespace Semmle.Extraction.CSharp
{
public class StandaloneAnalyser : Analyser
{
public StandaloneAnalyser(IProgressMonitor pm, ILogger logger, PathTransformer pathTransformer, IPathCache pathCache, bool addAssemblyTrapPrefix)
: base(pm, logger, pathTransformer, pathCache, addAssemblyTrapPrefix)
public StandaloneAnalyser(IProgressMonitor pm, ILogger logger, PathTransformer pathTransformer, IPathCache pathCache, IOverlayInfo overlayInfo, bool addAssemblyTrapPrefix)
: base(pm, logger, pathTransformer, pathCache, overlayInfo, addAssemblyTrapPrefix)
{
}

2
csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs
View File

@@ -14,7 +14,7 @@ namespace Semmle.Extraction.CSharp
private bool init;
public TracingAnalyser(IProgressMonitor pm, ILogger logger, PathTransformer pathTransformer, IPathCache pathCache, bool addAssemblyTrapPrefix)
: base(pm, logger, pathTransformer, pathCache, addAssemblyTrapPrefix)
: base(pm, logger, pathTransformer, pathCache, new TrivialOverlayInfo(), addAssemblyTrapPrefix)
{
}

8
csharp/extractor/Semmle.Extraction.CSharp/Populators/CommentPopulator.cs
View File

@@ -12,6 +12,10 @@ namespace Semmle.Extraction.CSharp.Populators
{
public static void ExtractCommentBlocks(Context cx, CommentProcessor gen)
{
if (cx.OnlyScaffold)
{
return;
}
cx.Try(null, null, () =>
{
gen.GenerateBindings((entity, duplicationGuardKey, block, binding) =>
@@ -34,6 +38,10 @@ namespace Semmle.Extraction.CSharp.Populators
public static void ExtractComment(Context cx, SyntaxTrivia trivia)
{
if (cx.OnlyScaffold)
{
return;
}
switch (trivia.Kind())
{
case SyntaxKind.SingleLineDocumentationCommentTrivia:

31
csharp/extractor/Semmle.Extraction.Tests/OverlayInfo.cs Normal file
View File

@@ -0,0 +1,31 @@
using Xunit;
using Semmle.Extraction.CSharp;
using System.IO;
namespace Semmle.Extraction.Tests
{
public class OverlayTests
{
[Fact]
public void TestOverlay()
{
var logger = new LoggerStub();
var json =
"""
{
"changes": [
"app/controllers/about_controller.xyz",
"app/models/about.xyz"
]
}
""";
var overlay = new OverlayInfo(logger, "overlay/source/path", json);
Assert.True(overlay.IsOverlayMode);
Assert.False(overlay.OnlyMakeScaffold("overlay/source/path" + Path.DirectorySeparatorChar + "app/controllers/about_controller.xyz"));
Assert.False(overlay.OnlyMakeScaffold("overlay/source/path" + Path.DirectorySeparatorChar + "app/models/about.xyz"));
Assert.True(overlay.OnlyMakeScaffold("overlay/source/path" + Path.DirectorySeparatorChar + "app/models/unchanged.xyz"));
}
}
}

23
csharp/extractor/Semmle.Util/EnvironmentVariables.cs
View File

@@ -53,5 +53,28 @@ namespace Semmle.Util
{
return Environment.GetEnvironmentVariable(name)?.Split(" ", StringSplitOptions.RemoveEmptyEntries) ?? [];
}
/// <summary>
/// Used to
/// (1) Detect whether the extractor should run in overlay mode.
/// (2) Returns the path to the file containing a list of changed files
/// in JSON format.
///
/// The environment variable is only set in case the extraction is supposed to be
/// performed in overlay mode. Furthermore, this only applies to buildless extraction.
/// </summary>
public static string? GetOverlayChangesFilePath()
{
return Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OVERLAY_CHANGES");
}
/// <summary>
/// If the environment variable is set, the extractor is being called to extract a base database.
/// Its value will be a path, and the extractor must create either a file or directory at that location.
/// </summary>
public static string? GetBaseMetaDataOutPath()
{
return Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OVERLAY_BASE_METADATA_OUT");
}
}
}