hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-27 01:35:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #7515 from hvitved/csharp/extraction-mode

Browse Source 
C#: Introduce extractor mode to identify DBs created with `codeql test run`
This commit is contained in:
Tom Hvitved
2022-01-19 16:04:57 +01:00
committed by GitHub
parent 162b3822dd 4f90b45dd7
commit 7e3f3c6e2a
24 changed files with 98 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
csharp/extractor/Semmle.Extraction.CIL.Driver/ExtractorOptions.cs
View File

@@ -6,6 +6,7 @@ using System.Reflection.Metadata;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Globalization;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CIL.Driver
@@ -169,18 +170,13 @@ namespace Semmle.Extraction.CIL.Driver
/// <summary>
/// Parses the command line and collates a list of DLLs/EXEs to extract.
/// </summary>
internal class ExtractorOptions
internal class ExtractorOptions : CommonOptions
{
private readonly AssemblyList assemblyList = new AssemblyList();
public ExtractorOptions(string[] args)
{
Verbosity = Verbosity.Info;
Threads = System.Environment.ProcessorCount;
PDB = true;
TrapCompression = TrapWriter.CompressionMode.Gzip;
ParseArgs(args);
this.ParseArguments(args.Append("--pdb").ToArray());
AddFrameworkDirectories(false);
@@ -203,12 +199,6 @@ namespace Semmle.Extraction.CIL.Driver
AddDirectory(RuntimeEnvironment.GetRuntimeDirectory(), extractAll);
}
public Verbosity Verbosity { get; private set; }
public bool NoCache { get; private set; }
public int Threads { get; private set; }
public bool PDB { get; private set; }
public TrapWriter.CompressionMode TrapCompression { get; private set; }
private void AddFileOrDirectory(string path)
{
path = Path.GetFullPath(path);
@@ -237,43 +227,25 @@ namespace Semmle.Extraction.CIL.Driver
/// </summary>
public IEnumerable<AssemblyName> MissingReferences => assemblyList.MissingReferences;
private void ParseArgs(string[] args)
public override bool HandleFlag(string flag, bool value)
{
foreach (var arg in args)
switch (flag)
{
if (arg == "--verbose")
{
Verbosity = Verbosity.All;
}
else if (arg == "--silent")
{
Verbosity = Verbosity.Off;
}
else if (arg.StartsWith("--verbosity:"))
{
Verbosity = (Verbosity)int.Parse(arg.Substring(12));
}
else if (arg == "--dotnet")
{
AddFrameworkDirectories(true);
}
else if (arg == "--nocache")
{
NoCache = true;
}
else if (arg.StartsWith("--threads:"))
{
Threads = int.Parse(arg.Substring(10));
}
else if (arg == "--no-pdb")
{
PDB = false;
}
else
{
AddFileOrDirectory(arg);
}
case "dotnet":
if (value)
AddFrameworkDirectories(true);
return true;
default:
return base.HandleFlag(flag, value);
}
}
public override bool HandleArgument(string argument)
{
AddFileOrDirectory(argument);
return true;
}
public override void InvalidArgument(string argument) { }
}
}

7
csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs
View File

@@ -20,11 +20,11 @@ namespace Semmle.Extraction.CIL.Driver
Console.WriteLine(" path A directory/dll/exe to analyze");
}
private static void ExtractAssembly(Layout layout, string assemblyPath, ILogger logger, bool nocache, bool extractPdbs, TrapWriter.CompressionMode trapCompression)
private static void ExtractAssembly(Layout layout, string assemblyPath, ILogger logger, CommonOptions options)
{
var sw = new Stopwatch();
sw.Start();
Analyser.ExtractCIL(layout, assemblyPath, logger, nocache, extractPdbs, trapCompression, out _, out _);
Analyser.ExtractCIL(layout, assemblyPath, logger, options, out _, out _);
sw.Stop();
logger.Log(Severity.Info, " {0} ({1})", assemblyPath, sw.Elapsed);
}
@@ -43,8 +43,7 @@ namespace Semmle.Extraction.CIL.Driver
var actions = options.AssembliesToExtract
.Select(asm => asm.Filename)
.Select<string, Action>(filename =>
() => ExtractAssembly(layout, filename, logger, options.NoCache, options.PDB, options.TrapCompression))
.Select<string, Action>(filename => () => ExtractAssembly(layout, filename, logger, options))
.ToArray();
foreach (var missingRef in options.MissingReferences)

10
csharp/extractor/Semmle.Extraction.CIL/Analyser.cs
View File

@@ -25,7 +25,7 @@ namespace Semmle.Extraction.CIL
/// <param name="extractPdbs">Whether to extract PDBs.</param>
/// <param name="trapFile">The path of the trap file.</param>
/// <param name="extracted">Whether the file was extracted (false=cached).</param>
public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger, bool nocache, bool extractPdbs, TrapWriter.CompressionMode trapCompression, out string trapFile, out bool extracted)
public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger, CommonOptions options, out string trapFile, out bool extracted)
{
trapFile = "";
extracted = false;
@@ -33,14 +33,14 @@ namespace Semmle.Extraction.CIL
{
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
var pathTransformer = new PathTransformer(canonicalPathCache);
var extractor = new TracingExtractor(assemblyPath, logger, pathTransformer);
var extractor = new TracingExtractor(assemblyPath, logger, pathTransformer, options);
var transformedAssemblyPath = pathTransformer.Transform(assemblyPath);
var project = layout.LookupProjectOrDefault(transformedAssemblyPath);
using var trapWriter = project.CreateTrapWriter(logger, transformedAssemblyPath.WithSuffix(".cil"), trapCompression, discardDuplicates: true);
using var trapWriter = project.CreateTrapWriter(logger, transformedAssemblyPath.WithSuffix(".cil"), options.TrapCompression, discardDuplicates: true);
trapFile = trapWriter.TrapFile;
if (nocache || !System.IO.File.Exists(trapFile))
if (!options.Cache || !System.IO.File.Exists(trapFile))
{
ExtractCIL(extractor, trapWriter, extractPdbs);
ExtractCIL(extractor, trapWriter, options.PDB);
extracted = true;
}
}

2
csharp/extractor/Semmle.Extraction.CIL/Entities/PdbSourceFile.cs
View File

@@ -25,7 +25,7 @@ namespace Semmle.Extraction.CIL.Entities
else
Context.TrapWriter.Archive(TransformedPath, text);
yield return Tuples.file_extraction_mode(this, 2);
yield return Tuples.file_extraction_mode(this, Context.Extractor.Mode | ExtractorMode.Pdb);
}
}
}

2
csharp/extractor/Semmle.Extraction.CIL/Tuples.cs
View File

@@ -206,7 +206,7 @@ namespace Semmle.Extraction.CIL
internal static Tuple files(File file, string fullName) =>
new Tuple("files", file, fullName);
internal static Tuple file_extraction_mode(File file, int mode) =>
internal static Tuple file_extraction_mode(File file, ExtractorMode mode) =>
new Tuple("file_extraction_mode", file, mode);
internal static Tuple folders(Folder folder, string path) =>

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/Assembly.cs
View File

@@ -64,7 +64,7 @@ namespace Semmle.Extraction.CSharp.Entities
public static Assembly CreateOutputAssembly(Context cx)
{
if (cx.Extractor.Standalone)
if (cx.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
throw new InternalError("Attempting to create the output assembly in standalone extraction mode");
return AssemblyConstructorFactory.Instance.CreateEntity(cx, outputAssemblyCacheKey, null);
}

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs
View File

@@ -53,7 +53,7 @@ namespace Semmle.Extraction.CSharp.Entities
if (attributeSyntax is not null)
{
if (!Context.Extractor.Standalone)
if (!Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
{
trapFile.attribute_location(this, Assembly.CreateOutputAssembly(Context));
}

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/CachedSymbol.cs
View File

@@ -85,7 +85,7 @@ namespace Semmle.Extraction.CSharp.Entities
{
// Some built in operators lack locations, so loc is null.
yield return Context.CreateLocation(ReportingLocation);
if (!Context.Extractor.Standalone && loc.Kind == LocationKind.SourceFile)
if (!Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone) && loc.Kind == LocationKind.SourceFile)
yield return Assembly.CreateOutputAssembly(Context);
}
}

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Invocation.cs
View File

@@ -129,7 +129,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
.Where(method => method.Parameters.Length >= Syntax.ArgumentList.Arguments.Count)
.Where(method => method.Parameters.Count(p => !p.HasExplicitDefaultValue) <= Syntax.ArgumentList.Arguments.Count);
return Context.Extractor.Standalone ?
return Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone) ?
candidates.FirstOrDefault() :
candidates.SingleOrDefault();
}

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/File.cs
View File

@@ -61,7 +61,7 @@ namespace Semmle.Extraction.CSharp.Entities
}
}
trapFile.file_extraction_mode(this, Context.Extractor.Standalone ? 1 : 0);
trapFile.file_extraction_mode(this, Context.Extractor.Mode);
}
private bool IsPossiblyTextFile()

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/PreprocessorDirective.cs
View File

@@ -16,7 +16,7 @@ namespace Semmle.Extraction.CSharp.Entities
trapFile.preprocessor_directive_active(this, Symbol.IsActive);
trapFile.preprocessor_directive_location(this, Context.CreateLocation(ReportingLocation));
if (!Context.Extractor.Standalone)
if (!Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
{
var compilation = Compilation.Create(Context);
trapFile.preprocessor_directive_compilation(this, compilation);

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs
View File

@@ -108,7 +108,7 @@ namespace Semmle.Extraction.CSharp.Entities
foreach (var l in GetLocations(Symbol))
yield return Context.CreateLocation(l);
if (!Context.Extractor.Standalone && Symbol.DeclaringSyntaxReferences.Any())
if (!Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone) && Symbol.DeclaringSyntaxReferences.Any())
yield return Assembly.CreateOutputAssembly(Context);
}
}

2
csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs
View File

@@ -178,7 +178,7 @@ namespace Semmle.Extraction.CSharp
{
var stopwatch = new Stopwatch();
stopwatch.Start();
CIL.Analyser.ExtractCIL(layout, r.FilePath!, Logger, !options.Cache, options.PDB, options.TrapCompression, out var trapFile, out var extracted);
CIL.Analyser.ExtractCIL(layout, r.FilePath!, Logger, options, out var trapFile, out var extracted);
stopwatch.Stop();
ReportProgress(r.FilePath, trapFile, stopwatch.Elapsed, extracted ? AnalysisAction.Extracted : AnalysisAction.UpToDate);
}

2
csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs
View File

@@ -15,7 +15,7 @@ namespace Semmle.Extraction.CSharp
{
compilation = compilationIn;
layout = new Layout();
extractor = new StandaloneExtractor(Logger, PathTransformer);
extractor = new StandaloneExtractor(Logger, PathTransformer, options);
this.options = options;
LogExtractorInfo(Extraction.Extractor.Version);
SetReferencePaths();

4
csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs
View File

@@ -49,7 +49,7 @@ namespace Semmle.Extraction.CSharp
this.layout = new Layout();
this.options = options;
this.compilation = compilation;
this.extractor = new TracingExtractor(GetOutputName(compilation, commandLineArguments), Logger, PathTransformer);
this.extractor = new TracingExtractor(GetOutputName(compilation, commandLineArguments), Logger, PathTransformer, options);
LogDiagnostics();
SetReferencePaths();
@@ -188,7 +188,7 @@ namespace Semmle.Extraction.CSharp
{
get
{
return extractor is null || extractor.Standalone || compilation is null ? Enumerable.Empty<Diagnostic>() :
return extractor is null || extractor.Mode.HasFlag(ExtractorMode.Standalone) || compilation is null ? Enumerable.Empty<Diagnostic>() :
compilation.
GetDiagnostics().
Where(e => e.Severity >= DiagnosticSeverity.Error && !errorsToIgnore.Contains(e.Id));

2
csharp/extractor/Semmle.Extraction.CSharp/Populators/TypeContainerVisitor.cs
View File

@@ -82,7 +82,7 @@ namespace Semmle.Extraction.CSharp.Populators
public override void VisitAttributeList(AttributeListSyntax node)
{
if (Cx.Extractor.Standalone)
if (Cx.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
return;
var outputAssembly = Assembly.CreateOutputAssembly(Cx);

2
csharp/extractor/Semmle.Extraction.CSharp/Tuples.cs
View File

@@ -451,7 +451,7 @@ namespace Semmle.Extraction.CSharp
internal static void locations_mapped(this System.IO.TextWriter trapFile, NonGeneratedSourceLocation l1, Location l2) =>
trapFile.WriteTuple("locations_mapped", l1, l2);
internal static void file_extraction_mode(this System.IO.TextWriter trapFile, Entities.File file, int mode) =>
internal static void file_extraction_mode(this System.IO.TextWriter trapFile, Entities.File file, ExtractorMode mode) =>
trapFile.WriteTuple("file_extraction_mode", file, mode);
}
}

2
csharp/extractor/Semmle.Extraction/Context.cs
View File

@@ -400,7 +400,7 @@ namespace Semmle.Extraction
private void ReportError(InternalError error)
{
if (!Extractor.Standalone)
if (!Extractor.Mode.HasFlag(ExtractorMode.Standalone))
throw error;
ExtractionError(error);

2
csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs
View File

@@ -8,7 +8,7 @@ namespace Semmle.Extraction
/// </summary>
public abstract class Extractor
{
public abstract bool Standalone { get; }
public abstract ExtractorMode Mode { get; }
/// <summary>
/// Creates a new extractor instance for one compilation unit.

16
csharp/extractor/Semmle.Extraction/Extractor/ExtractorMode.cs Normal file
View File

@@ -0,0 +1,16 @@
using System;
namespace Semmle.Extraction
{
/// <summary>
/// The mode in which a file is extracted.
/// </summary>
[Flags]
public enum ExtractorMode
{
None = 0,
Standalone = 1,
Pdb = 2,
QlTest = 4,
}
}

9
csharp/extractor/Semmle.Extraction/Extractor/StandaloneExtractor.cs
View File

@@ -4,15 +4,20 @@ namespace Semmle.Extraction
{
public class StandaloneExtractor : Extractor
{
public override bool Standalone => true;
public override ExtractorMode Mode { get; }
/// <summary>
/// Creates a new extractor instance for one compilation unit.
/// </summary>
/// <param name="logger">The object used for logging.</param>
/// <param name="pathTransformer">The object used for path transformations.</param>
public StandaloneExtractor(ILogger logger, PathTransformer pathTransformer) : base(logger, pathTransformer)
public StandaloneExtractor(ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(logger, pathTransformer)
{
Mode = ExtractorMode.Standalone;
if (options.QlTest)
{
Mode |= ExtractorMode.QlTest;
}
}
}
}

10
csharp/extractor/Semmle.Extraction/Extractor/TracingExtractor.cs
View File

@@ -4,8 +4,7 @@ namespace Semmle.Extraction
{
public class TracingExtractor : Extractor
{
public override bool Standalone => false;
public override ExtractorMode Mode { get; }
public string OutputPath { get; }
/// <summary>
@@ -14,9 +13,14 @@ namespace Semmle.Extraction
/// <param name="outputPath">The name of the output DLL/EXE, or null if not specified (standalone extraction).</param>
/// <param name="logger">The object used for logging.</param>
/// <param name="pathTransformer">The object used for path transformations.</param>
public TracingExtractor(string outputPath, ILogger logger, PathTransformer pathTransformer) : base(logger, pathTransformer)
public TracingExtractor(string outputPath, ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(logger, pathTransformer)
{
OutputPath = outputPath;
Mode = ExtractorMode.None;
if (options.QlTest)
{
Mode |= ExtractorMode.QlTest;
}
}
}
}

12
csharp/extractor/Semmle.Extraction/Options.cs
View File

@@ -44,6 +44,11 @@ namespace Semmle.Extraction
/// </summary>
public bool Fast { get; private set; } = false;
/// <summary>
/// Whether extraction is done using `codeql test run`.
/// </summary>
public bool QlTest { get; private set; } = false;
/// <summary>
/// The compression algorithm used for trap files.
/// </summary>
@@ -70,6 +75,10 @@ namespace Semmle.Extraction
{
switch (flag)
{
case "silent":
if (value)
Verbosity = Verbosity.Off;
return true;
case "verbose":
Verbosity = value ? Verbosity.Debug : Verbosity.Error;
return true;
@@ -90,6 +99,9 @@ namespace Semmle.Extraction
CIL = !value;
Fast = value;
return true;
case "qltest":
QlTest = value;
return true;
case "brotli":
TrapCompression = value ? TrapWriter.CompressionMode.Brotli : TrapWriter.CompressionMode.Gzip;
return true;

14
csharp/ql/lib/semmle/code/csharp/File.qll
View File

@@ -218,7 +218,12 @@ class File extends Container, @file {
* A source file can come from a PDB and from regular extraction
* in the same snapshot.
*/
predicate isPdbSourceFile() { file_extraction_mode(this, 2) }
predicate isPdbSourceFile() {
exists(int i |
file_extraction_mode(this, i) and
i.bitAnd(2) = 2
)
}
}
/**
@@ -228,5 +233,10 @@ class SourceFile extends File {
SourceFile() { this.fromSource() }
/** Holds if the file was extracted without building the source code. */
predicate extractedStandalone() { file_extraction_mode(this, 1) }
predicate extractedStandalone() {
exists(int i |
file_extraction_mode(this, i) and
i.bitAnd(1) = 1
)
}
}