hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-25 16:55:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #16938 from tamasvajk/feature/extract-files-multiple

Browse Source 
C#: Do not skip extraction of already seen source files
This commit is contained in:
Tamás Vajk
2024-08-07 11:23:48 +02:00
committed by GitHub
parent 62c2fe6b17 a4e357e46d
commit 42be9e98c8
19 changed files with 8514 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2099
csharp/downgrades/15b989afd2bfc4743536fdb0958c1d8177a32600/old.dbscheme Normal file
View File

File diff suppressed because it is too large Load Diff

2099
csharp/downgrades/15b989afd2bfc4743536fdb0958c1d8177a32600/semmlecode.csharp.dbscheme Normal file
View File

File diff suppressed because it is too large Load Diff

2
csharp/downgrades/15b989afd2bfc4743536fdb0958c1d8177a32600/upgrade.properties Normal file
View File

@@ -0,0 +1,2 @@
description: Add unique constraint on preprocessor directive and compilation pairs
compatibility: backwards

1
csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/ElifDirective.cs
View File

@@ -20,6 +20,7 @@ namespace Semmle.Extraction.CSharp.Entities
public override void WriteId(EscapingTextWriter trapFile)
{
trapFile.WriteSubId(Context.CreateLocation(ReportingLocation));
trapFile.WriteSubId(start);
trapFile.Write(Symbol.IsActive);
trapFile.Write(',');
trapFile.Write(Symbol.BranchTaken);

1
csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/ElseDirective.cs
View File

@@ -18,6 +18,7 @@ namespace Semmle.Extraction.CSharp.Entities
public override void WriteId(EscapingTextWriter trapFile)
{
trapFile.WriteSubId(Context.CreateLocation(ReportingLocation));
trapFile.WriteSubId(start);
trapFile.Write(Symbol.IsActive);
trapFile.Write(',');
trapFile.Write(Symbol.BranchTaken);

8
csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/EndIfDirective.cs
View File

@@ -13,6 +13,14 @@ namespace Semmle.Extraction.CSharp.Entities
this.start = start;
}
public override void WriteId(EscapingTextWriter trapFile)
{
trapFile.WriteSubId(Context.CreateLocation(ReportingLocation));
trapFile.WriteSubId(start);
trapFile.Write(Symbol.IsActive);
trapFile.Write(";trivia");
}
protected override void PopulatePreprocessor(TextWriter trapFile)
{
trapFile.directive_endifs(this, start);

44
csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs
View File

@@ -190,37 +190,29 @@ namespace Semmle.Extraction.CSharp
var transformedSourcePath = PathTransformer.Transform(sourcePath);
var trapPath = transformedSourcePath.GetTrapPath(Logger, options.TrapCompression);
var upToDate = false;
// compilation.Clone() is used to allow symbols to be garbage collected.
using var trapWriter = transformedSourcePath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
upToDate = FileIsUpToDate(sourcePath, trapWriter.TrapFile);
var currentTaskId = IncrementTaskCount();
ReportProgressTaskStarted(currentTaskId, sourcePath);
if (!upToDate)
var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
// Ensure that the file itself is populated in case the source file is totally empty
var root = tree.GetRoot();
Entities.File.Create(cx, root.SyntaxTree.FilePath);
var csNode = (CSharpSyntaxNode)root;
var directiveVisitor = new DirectiveVisitor(cx);
csNode.Accept(directiveVisitor);
foreach (var branch in directiveVisitor.BranchesTaken)
{
var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
// Ensure that the file itself is populated in case the source file is totally empty
var root = tree.GetRoot();
Entities.File.Create(cx, root.SyntaxTree.FilePath);
var csNode = (CSharpSyntaxNode)root;
var directiveVisitor = new DirectiveVisitor(cx);
csNode.Accept(directiveVisitor);
foreach (var branch in directiveVisitor.BranchesTaken)
{
cx.TrapStackSuffix.Add(branch);
}
csNode.Accept(new CompilationUnitVisitor(cx));
cx.PopulateAll();
CommentPopulator.ExtractCommentBlocks(cx, cx.CommentGenerator);
cx.PopulateAll();
cx.TrapStackSuffix.Add(branch);
}
csNode.Accept(new CompilationUnitVisitor(cx));
cx.PopulateAll();
CommentPopulator.ExtractCommentBlocks(cx, cx.CommentGenerator);
cx.PopulateAll();
ReportProgressTaskDone(currentTaskId, sourcePath, trapPath, stopwatch.Elapsed, upToDate ? AnalysisAction.UpToDate : AnalysisAction.Extracted);
ReportProgressTaskDone(currentTaskId, sourcePath, trapPath, stopwatch.Elapsed, AnalysisAction.Extracted);
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
@@ -268,12 +260,6 @@ namespace Semmle.Extraction.CSharp
extractionTasks.Add(() => DoAnalyseCompilation());
}
private static bool FileIsUpToDate(string src, string dest)
{
return File.Exists(dest) &&
File.GetLastWriteTime(dest) >= File.GetLastWriteTime(src);
}
private static void AnalyseNamespace(Context cx, INamespaceSymbol ns)
{
foreach (var memberNamespace in ns.GetNamespaceMembers())

15
csharp/extractor/Semmle.Extraction/TrapWriter.cs
View File

@@ -194,8 +194,11 @@ namespace Semmle.Extraction
var hash = FileUtils.ComputeFileHash(tmpFile);
if (existingHash != hash)
{
var root = TrapFile.Substring(0, TrapFile.Length - 8); // Remove trailing ".trap.gz"
if (TryMove(tmpFile, $"{root}-{hash}.trap{TrapExtension(trapCompression)}"))
var extension = TrapExtension(trapCompression);
var root = TrapFile[..^extension.Length]; // Remove trailing ".trap", ".trap.gz", or ".trap.br"
var newTrapName = $"{root}-{hash}{extension}";
logger.LogInfo($"Identical trap file for {TrapFile} already exists, renaming to {newTrapName}");
if (TryMove(tmpFile, $"{newTrapName}"))
return;
}
logger.LogInfo($"Identical trap file for {TrapFile} already exists");
@@ -217,16 +220,16 @@ namespace Semmle.Extraction
{
switch (compression)
{
case CompressionMode.None: return "";
case CompressionMode.Gzip: return ".gz";
case CompressionMode.Brotli: return ".br";
case CompressionMode.None: return ".trap";
case CompressionMode.Gzip: return ".trap.gz";
case CompressionMode.Brotli: return ".trap.br";
default: throw new ArgumentOutOfRangeException(nameof(compression), compression, "Unsupported compression type");
}
}
public static string TrapPath(ILogger logger, string? folder, PathTransformer.ITransformedPath path, TrapWriter.CompressionMode trapCompression)
{
var filename = $"{path.Value}.trap{TrapExtension(trapCompression)}";
var filename = $"{path.Value}{TrapExtension(trapCompression)}";
if (string.IsNullOrEmpty(folder))
folder = Directory.GetCurrentDirectory();

4
csharp/ql/integration-tests/all-platforms/conditional_compilation/Methods.expected Normal file
View File

@@ -0,0 +1,4 @@
| Program.cs:3:24:3:27 | Main | 2 |
| Program.cs:10:17:10:33 | GreetConditional1 | 2 |
| Program.cs:19:17:19:21 | Greet | 2 |
| Program.cs:25:17:25:33 | GreetConditional2 | 2 |

5
csharp/ql/integration-tests/all-platforms/conditional_compilation/Methods.ql Normal file
View File

@@ -0,0 +1,5 @@
import csharp
from Method m
where m.fromSource()
select m, count(m.getBody())

40
csharp/ql/integration-tests/all-platforms/conditional_compilation/Program.cs Normal file
View File

@@ -0,0 +1,40 @@
public class Test
{
public static void Main()
{
Greet();
GreetConditional1();
GreetConditional2();
}
static void GreetConditional1()
{
#if A
Console.WriteLine("Hello, A!");
#elif C
Console.WriteLine("Hello, C!");
#endif
}
static void Greet()
{
Console.WriteLine("Hello, World!");
}
#if A
static void GreetConditional2()
{
Console.WriteLine("Hello, A!");
}
#elif B
static void GreetConditional2()
{
Console.WriteLine("Hello, B!");
}
#else
static void GreetConditional2()
{
Console.WriteLine("Hello, Others!");
}
#endif
}

5
csharp/ql/integration-tests/all-platforms/conditional_compilation/global.json Normal file
View File

@@ -0,0 +1,5 @@
{
"sdk": {
"version": "8.0.101"
}
}

15
csharp/ql/integration-tests/all-platforms/conditional_compilation/test.csproj Normal file
View File

@@ -0,0 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<Compile Remove="**/*.cs" />
<Compile Include="Program.cs" />
</ItemGroup>
</Project>

6
csharp/ql/integration-tests/all-platforms/conditional_compilation/test.py Normal file
View File

@@ -0,0 +1,6 @@
from create_database_utils import *
import os
os.environ["CODEQL_EXTRACTOR_CSHARP_OPTION_TRAP_COMPRESSION"] = "none"
run_codeql_database_create(['dotnet build /p:DefineConstants=A', 'dotnet build /p:DefineConstants=B'], lang="csharp")

4
csharp/ql/lib/change-notes/2024-07-10-conditional-compilation.md Normal file
View File

@@ -0,0 +1,4 @@
---
category: minorAnalysis
---
* The extractor has been changed to not skip source files that have already been seen. This has an impact on source files that are compiled multiple times in the build process. Source files with conditional compilation preprocessor directives (such as `#if`) are now extracted for each set of preprocessor symbols that are used during the build process.

2
csharp/ql/lib/semmlecode.csharp.dbscheme
View File

@@ -421,7 +421,7 @@ preprocessor_directive_location(
int loc: @location ref);
preprocessor_directive_compilation(
unique int id: @preprocessor_directive ref,
int id: @preprocessor_directive ref,
int compilation: @compilation ref);
preprocessor_directive_active(

2099
csharp/ql/lib/upgrades/fd04e45710e1988076801608abffdfa013b680fc/old.dbscheme Normal file
View File

File diff suppressed because it is too large Load Diff

2099
csharp/ql/lib/upgrades/fd04e45710e1988076801608abffdfa013b680fc/semmlecode.csharp.dbscheme Normal file
View File

File diff suppressed because it is too large Load Diff

2
csharp/ql/lib/upgrades/fd04e45710e1988076801608abffdfa013b680fc/upgrade.properties Normal file
View File

@@ -0,0 +1,2 @@
description: Remove unique constraint on preprocessor directive and compilation pairs
compatibility: backwards