C#: Store compilations, compiler diagnostics and performance in the database.

This commit is contained in:
calum
2019-03-21 17:57:27 +00:00
committed by Calum Grant
parent 4cc23cce13
commit e1158bb5e5
17 changed files with 484 additions and 15 deletions

View File

@@ -139,7 +139,7 @@ namespace Semmle.Extraction.CIL.Entities
trapFile = trapWriter.TrapFile;
if (nocache || !System.IO.File.Exists(trapFile))
{
var cx = new Extraction.Context(extractor, null, trapWriter, null);
var cx = extractor.CreateContext(null, trapWriter, null);
ExtractCIL(cx, assemblyPath, extractPdbs);
extracted = true;
}

View File

@@ -215,6 +215,38 @@ namespace Semmle.Extraction.CSharp
return options.Cache && FileIsUpToDate(src, dest);
}
/// <summary>
/// Extracts compilation-wide entities, such as compilations and compiler diagnostics.
/// </summary>
public void AnalyseCompilation(string cwd, string[] args)
{
extractionTasks.Add(() => DoAnalyseCompilation(cwd, args));
}
Entities.Compilation compilationEntity;
IDisposable compilationTrapFile;
void DoAnalyseCompilation(string cwd, string[] args)
{
try
{
var assemblyPath = extractor.OutputPath;
var assembly = compilation.Assembly;
var projectLayout = layout.LookupProjectOrDefault(assemblyPath);
var trapWriter = projectLayout.CreateTrapWriter(Logger, assemblyPath, true);
compilationTrapFile = trapWriter; // Dispose later
var cx = extractor.CreateContext(compilation.Clone(), trapWriter, new AssemblyScope(assembly, assemblyPath, true));
compilationEntity = new Entities.Compilation(cx, cwd, args);
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
Logger.Log(Severity.Error, " Unhandled exception analyzing {0}: {1}", "compilation", ex);
}
}
public void LogPerformance(Entities.Performance p) => compilationEntity.PopulatePerformance(p);
/// <summary>
/// Extract an assembly to a new trap file.
/// If the trap file exists, skip extraction to avoid duplicating
@@ -258,7 +290,7 @@ namespace Semmle.Extraction.CSharp
if (assembly != null)
{
var cx = new Context(extractor, c, trapWriter, new AssemblyScope(assembly, assemblyPath));
var cx = extractor.CreateContext(c, trapWriter, new AssemblyScope(assembly, assemblyPath, false));
foreach (var module in assembly.Modules)
{
@@ -344,7 +376,7 @@ namespace Semmle.Extraction.CSharp
if (!upToDate)
{
Context cx = new Context(extractor, compilation.Clone(), trapWriter, new SourceScope(tree));
Context cx = extractor.CreateContext(compilation.Clone(), trapWriter, new SourceScope(tree));
Populators.CompilationUnit.Extract(cx, tree.GetRoot());
cx.PopulateAll();
cx.ExtractComments(cx.CommentGenerator);
@@ -373,6 +405,8 @@ namespace Semmle.Extraction.CSharp
public void Dispose()
{
compilationTrapFile?.Dispose();
stopWatch.Stop();
Logger.Log(Severity.Info, " Peak working set = {0} MB", Process.GetCurrentProcess().PeakWorkingSet64 / (1024 * 1024));

View File

@@ -0,0 +1,94 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
namespace Semmle.Extraction.CSharp.Entities
{
class Compilation : FreshEntity
{
public Compilation(Context cx, string cwd, string[] args) : base(cx)
{
Extraction.Entities.Assembly.CreateOutputAssembly(cx);
cx.Emit(Tuples.compilations(this, Extraction.Entities.File.PathAsDatabaseString(cwd)));
// Arguments
int index = 0;
foreach(var arg in args)
{
cx.Emit(Tuples.compilation_args(this, index, args[index]));
index++;
}
// Files
index = 0;
foreach(var file in cx.Compilation.SyntaxTrees.Select(tree => Extraction.Entities.File.Create(cx, tree.FilePath)))
{
cx.Emit(Tuples.compilation_compiling_files(this, index++, file));
}
// Diagnostics
index = 0;
foreach(var d in cx.Compilation.GetDiagnostics())
{
var d2 = new Diagnostic(cx, d);
cx.Emit(Tuples.diagnostic_for(d2, this, 0, index++));
}
}
public void PopulatePerformance(Performance p)
{
int index = 0;
foreach(float metric in p.Metrics)
{
cx.Emit(Tuples.compilation_time(this, -1, index++, metric));
}
}
public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel;
}
class Diagnostic : FreshEntity
{
public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel;
public Diagnostic(Context cx, Microsoft.CodeAnalysis.Diagnostic diag) : base(cx)
{
cx.Emit(Tuples.diagnostics(this, (int)diag.Severity, diag.Id, diag.Descriptor.Title.ToString(),
diag.GetMessage(), Extraction.Entities.Location.Create(cx, diag.Location)));
}
}
public struct Timings
{
public TimeSpan Elapsed, Cpu, User;
}
/// <summary>
/// The various performance measures that we log.
/// </summary>
public struct Performance
{
public Timings Compiler, Extractor;
public long PeakWorkingSet;
/// <summary>
/// These are in database order (0 indexed)
/// </summary>
public IEnumerable<float> Metrics
{
get
{
yield return (float)Compiler.Cpu.TotalSeconds;
yield return (float)Compiler.Elapsed.TotalSeconds;
yield return (float)Extractor.Cpu.TotalSeconds;
yield return (float)Extractor.Elapsed.TotalSeconds;
yield return (float)Compiler.User.TotalSeconds;
yield return (float)Extractor.User.TotalSeconds;
yield return PeakWorkingSet / 1024.0f / 1024.0f;
}
}
}
}

View File

@@ -118,8 +118,8 @@ namespace Semmle.Extraction.CSharp
compilerArguments.Encoding,
syntaxTrees);
var sw = new Stopwatch();
sw.Start();
var sw1 = new Stopwatch();
sw1.Start();
Parallel.Invoke(
new ParallelOptions { MaxDegreeOfParallelism = commandLineArguments.Threads },
@@ -148,6 +148,7 @@ namespace Semmle.Extraction.CSharp
);
analyser.Initialize(compilerArguments, compilation, commandLineArguments, compilerVersion.ArgsWithResponse);
analyser.AnalyseCompilation(cwd, args);
analyser.AnalyseReferences();
foreach (var tree in compilation.SyntaxTrees)
@@ -155,13 +156,28 @@ namespace Semmle.Extraction.CSharp
analyser.AnalyseTree(tree);
}
sw.Stop();
logger.Log(Severity.Info, " Models constructed in {0}", sw.Elapsed);
var currentProcess = Process.GetCurrentProcess();
var cpuTime1 = currentProcess.TotalProcessorTime;
var userTime1 = currentProcess.UserProcessorTime;
sw1.Stop();
logger.Log(Severity.Info, " Models constructed in {0}", sw1.Elapsed);
sw.Restart();
var sw2 = new Stopwatch();
sw2.Start();
analyser.PerformExtraction(commandLineArguments.Threads);
sw.Stop();
logger.Log(Severity.Info, " Extraction took {0}", sw.Elapsed);
sw2.Stop();
var cpuTime2 = currentProcess.TotalProcessorTime;
var userTime2 = currentProcess.UserProcessorTime;
var performance = new Entities.Performance()
{
Compiler = new Entities.Timings() { Elapsed = sw1.Elapsed, Cpu = cpuTime1, User = userTime1 },
Extractor = new Entities.Timings() { Elapsed = sw2.Elapsed, Cpu = cpuTime2 - cpuTime1, User = userTime2 - userTime1 },
PeakWorkingSet = currentProcess.PeakWorkingSet64
};
analyser.LogPerformance(performance);
logger.Log(Severity.Info, " Extraction took {0}", sw2.Elapsed);
return analyser.TotalErrors == 0 ? ExitCode.Ok : ExitCode.Errors;
}

View File

@@ -41,6 +41,14 @@ namespace Semmle.Extraction.CSharp
internal static Tuple commentline_location(CommentLine commentLine, Location location) => new Tuple("commentline_location", commentLine, location);
internal static Tuple compilation_args(Compilation compilation, int index, string arg) => new Tuple("compilation_args", compilation, index, arg);
internal static Tuple compilation_compiling_files(Compilation compilation, int index, File file) => new Tuple("compilation_compiling_files", compilation, index, file);
internal static Tuple compilation_time(Compilation compilation, int num, int index, float metric) => new Tuple("compilation_time", compilation, num, index, metric);
internal static Tuple compilations(Compilation compilation, string cwd) => new Tuple("compilations", compilation, cwd);
internal static Tuple compiler_generated(IEntity entity) => new Tuple("compiler_generated", entity);
internal static Tuple conditional_access(Expression access) => new Tuple("conditional_access", access);
@@ -59,6 +67,11 @@ namespace Semmle.Extraction.CSharp
internal static Tuple destructors(Destructor destructor, string name, Type containingType, Destructor original) => new Tuple("destructors", destructor, name, containingType, original);
internal static Tuple diagnostic_for(Diagnostic diag, Compilation comp, int fileNo, int index) => new Tuple("diagnostic_for", diag, comp, fileNo, index);
internal static Tuple diagnostics(Diagnostic diag, int severity, string errorTag, string errorMessage, string fullErrorMessage, Location location) =>
new Tuple("diagnostics", diag, severity, errorTag, errorMessage, fullErrorMessage, location);
internal static Tuple dynamic_member_name(Expression e, string name) => new Tuple("dynamic_member_name", e, name);
internal static Tuple enum_underlying_type(Type @enum, Type type) => new Tuple("enum_underlying_type", @enum, type);

View File

@@ -247,6 +247,8 @@ namespace Semmle.Extraction
TrapWriter = trapWriter;
}
public bool IsGlobalContext => Scope.IsGlobalScope;
public readonly ICommentGenerator CommentGenerator = new CommentProcessor();
readonly IExtractionScope Scope;

View File

@@ -34,7 +34,8 @@ namespace Semmle.Extraction.Entities
}
}
public override bool NeedsPopulation => true;
public override bool NeedsPopulation =>
assembly != Context.Compilation.Assembly || !Context.IsGlobalContext;
public override int GetHashCode() =>
symbol == null ? 91187354 : symbol.GetHashCode();

View File

@@ -27,7 +27,7 @@ namespace Semmle.Extraction.Entities
// On Windows: System.IO.DirectoryInfo.Name returns "L:\"
string shortName = symbol.Parent == null ? "" : symbol.Name;
Context.Emit(Tuples.folders(this, Semmle.Extraction.Entities.File.PathAsDatabaseString(Path), shortName));
Context.Emit(Tuples.folders(this, File.PathAsDatabaseString(Path), shortName));
if (symbol.Parent != null)
{
Context.Emit(Tuples.containerparent(Create(Context, symbol.Parent), this));

View File

@@ -6,7 +6,7 @@ namespace Semmle.Extraction.Entities
public Location(Context cx, Microsoft.CodeAnalysis.Location init)
: base(cx, init) { }
internal static Location Create(Context cx, Microsoft.CodeAnalysis.Location loc) =>
public static Location Create(Context cx, Microsoft.CodeAnalysis.Location loc) =>
loc == null ? GeneratedLocation.Create(cx)
: loc.IsInSource ? SourceLocation.Create(cx, loc)
: Assembly.Create(cx, loc);

View File

@@ -23,6 +23,8 @@ namespace Semmle.Extraction
/// </summary>
/// <param name="path">The path to populate.</param>
bool InFileScope(string path);
bool IsGlobalScope { get; }
}
/// <summary>
@@ -33,15 +35,18 @@ namespace Semmle.Extraction
readonly IAssemblySymbol assembly;
readonly string filepath;
public AssemblyScope(IAssemblySymbol symbol, string path)
public AssemblyScope(IAssemblySymbol symbol, string path, bool isOutput)
{
assembly = symbol;
filepath = path;
IsGlobalScope = isOutput;
}
public bool IsGlobalScope { get; }
public bool InFileScope(string path) => path == filepath;
public bool InScope(ISymbol symbol) => Equals(symbol.ContainingAssembly, assembly);
public bool InScope(ISymbol symbol) => Equals(symbol.ContainingAssembly, assembly) || Equals(symbol, assembly);
}
/// <summary>
@@ -56,6 +61,8 @@ namespace Semmle.Extraction
sourceTree = tree;
}
public bool IsGlobalScope => false;
public bool InFileScope(string path) => path == sourceTree.FilePath;
public bool InScope(ISymbol symbol) => symbol.Locations.Any(loc => loc.SourceTree == sourceTree);

View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using Microsoft.CodeAnalysis;
using Semmle.Util;
using Semmle.Util.Logging;
@@ -82,6 +83,15 @@ namespace Semmle.Extraction
/// The extractor SHA, obtained from the git log.
/// </summary>
string Version { get; }
/// <summary>
/// Creates a new context.
/// </summary>
/// <param name="c">The C# compilation.</param>
/// <param name="trapWriter">The trap writer.</param>
/// <param name="scope">The extraction scope (what to include in this trap file).</param>
/// <returns></returns>
Context CreateContext(Compilation c, TrapWriter trapWriter, IExtractionScope scope);
}
/// <summary>
@@ -191,6 +201,11 @@ namespace Semmle.Extraction
missingNamespaces.Add(fqdn);
}
public Context CreateContext(Compilation c, TrapWriter trapWriter, IExtractionScope scope)
{
return new Context(this, c, trapWriter, scope);
}
public IEnumerable<string> MissingTypes => missingTypes;
public IEnumerable<string> MissingNamespaces => missingNamespaces;

View File

@@ -120,6 +120,9 @@ namespace Semmle.Extraction
case int i:
tb.Append(i);
break;
case float f:
tb.Append(f.ToString("0.#####e0")); // Trap importer won't accept ints
break;
case string[] array:
tb.Append("\"");
if (NeedsTruncation(array))

View File

@@ -0,0 +1,78 @@
/** Provides classes relating to compilation and extraction diagnostics. */
import csharp
/** An invocation of the C# compiler. */
class Compilation extends @compilation {
/** Gets a textual representation of this compilation. */
string toString() { result = "compilation" }
/** Gets the directory in which this compilation was run, as a string. */
string getDirectoryString() { compilations(this, result) }
/** Gets the folder in which this compilation was run. */
Folder getFolder() { result.getAbsolutePath() = getDirectoryString() }
/** Gets the `i`th command line argument. */
string getArgument(int i) { compilation_args(this, i, result) }
/** Gets the 'i'th source file in this compilation. */
File getFile(int i) { compilation_compiling_files(this, i, result) }
/** Gets a diagnostic associated with this compilation. */
Diagnostic getADiagnostic() { result.getCompilation() = this }
/** Gets a performance metric for this compilation. */
float getMetric(int metric) { compilation_time(this, -1, metric, result) }
/** Gets the CPU time of the compilation. */
float getCompilationCpuTime() { result = getMetric(0) }
/** Gets the elapsed time of the compilation. */
float getCompilationElapsedTime() { result = getMetric(1) }
/** Gets the CPU time of the extraction. */
float getExtractionCpuTime() { result = getMetric(2) }
/** Gets the elapsed time of the extraction. */
float getExtractionElapsedTime() { result = getMetric(3) }
/** Gets the user CPU time of the compilation. */
float getCompilationUserCpuTime() { result = getMetric(4) }
/** Gets the user CPU time of the extraction. */
float getExtractionUserCpuTime() { result = getMetric(5) }
/** Gets the peak working set of the extractor process in MB. */
float getPeakWorkingSetMB() { result = getMetric(6) }
}
/** A diagnostic emitted by a compilation, such as a compilation warning or an error. */
class Diagnostic extends @diagnostic {
/** Gets the compilation that generated this diagnostic. */
Compilation getCompilation() { diagnostic_for(this, result, _, _) }
/**
* Gets the severity of this diagnostic.
* 0 = Hidden
* 1 = Info
* 2 = Warning
* 3 = Error
*/
int getSeverity() { diagnostics(this, result, _, _, _, _) }
/** Gets the identifier of this diagnostic, for example "CS8019". */
string getId() { diagnostics(this, _, result, _, _, _) }
/** Gets the short error message of this diagnostic. */
string getMessage() { diagnostics(this, _, _, result, _, _) }
/** Gets the full error message of this diagnostic. */
string getFullMessage() { diagnostics(this, _, _, _, result, _) }
/** Gets the location of this diagnostic. */
Location getLocation() { diagnostics(this, _, _, _, _, result) }
/** Gets a textual representation of this diagnostic. */
string toString() { result = getMessage() }
}

View File

@@ -1,3 +1,146 @@
/**
* An invocation of the compiler. Note that more than one file may be
* compiled per invocation. For example, this command compiles three
* source files:
*
* gcc -c f1.c f2.c f3.c
*
* The `id` simply identifies the invocation, while `cwd` is the working
* directory from which the compiler was invoked.
*/
compilations(
/**
* An invocation of the compiler. Note that more than one file may
* be compiled per invocation. For example, this command compiles
* three source files:
*
* gcc -c f1.c f2.c f3.c
*/
unique int id : @compilation,
string cwd : string ref
);
/**
* The arguments that were passed to the extractor for a compiler
* invocation. If `id` is for the compiler invocation
*
* gcc -c f1.c f2.c f3.c
*
* then typically there will be rows for
*
* num | arg
* --- | ---
* 0 | *path to extractor*
* 1 | `--mimic`
* 2 | `/usr/bin/gcc`
* 3 | `-c`
* 4 | f1.c
* 5 | f2.c
* 6 | f3.c
*/
#keyset[id, num]
compilation_args(
int id : @compilation ref,
int num : int ref,
string arg : string ref
);
/**
* The source files that are compiled by a compiler invocation.
* If `id` is for the compiler invocation
*
* gcc -c f1.c f2.c f3.c
*
* then there will be rows for
*
* num | arg
* --- | ---
* 0 | f1.c
* 1 | f2.c
* 2 | f3.c
*
* Note that even if those files `#include` headers, those headers
* do not appear as rows.
*/
#keyset[id, num]
compilation_compiling_files(
int id : @compilation ref,
int num : int ref,
int file : @file ref
);
/**
* The time taken by the extractor for a compiler invocation.
*
* For each file `num`, there will be rows for
*
* kind | seconds
* ---- | ---
* 1 | CPU seconds used by the extractor frontend
* 2 | Elapsed seconds during the extractor frontend
* 3 | CPU seconds used by the extractor backend
* 4 | Elapsed seconds during the extractor backend
*/
#keyset[id, num, kind]
compilation_time(
int id : @compilation ref,
int num : int ref,
/* kind:
1 = frontend_cpu_seconds
2 = frontend_elapsed_seconds
3 = extractor_cpu_seconds
4 = extractor_elapsed_seconds
*/
int kind : int ref,
float seconds : float ref
);
/**
* An error or warning generated by the extractor.
* The diagnostic message `diagnostic` was generated during compiler
* invocation `compilation`, and is the `file_number_diagnostic_number`th
* message generated while extracting the `file_number`th file of that
* invocation.
*/
#keyset[compilation, file_number, file_number_diagnostic_number]
diagnostic_for(
unique int diagnostic : @diagnostic ref,
int compilation : @compilation ref,
int file_number : int ref,
int file_number_diagnostic_number : int ref
);
diagnostics(
unique int id: @diagnostic,
int severity: int ref,
string error_tag: string ref,
string error_message: string ref,
string full_error_message: string ref,
int location: @location_default ref
);
extraction_errors(
unique int id: @extraction_error,
int compilation: @compilation ref,
int element: @element ref,
int severity: int ref,
string error_tag: string ref,
string error_message : string ref,
string stack_trace : string ref
);
/**
* If extraction was successful, then `cpu_seconds` and
* `elapsed_seconds` are the CPU time and elapsed time (respectively)
* that extraction took for compiler invocation `id`.
*/
compilation_finished(
unique int id : @compilation ref,
float cpu_seconds : float ref,
float elapsed_seconds : float ref
);
/*
* External artifacts
*/

View File

@@ -0,0 +1,26 @@
diags
| Program.cs:1:1:1:13 | Unnecessary using directive | CS8019 | 0 | Unnecessary using directive | Unnecessary using directive. |
| Program.cs:7:13:7:13 | Variable is assigned but its value is never used | CS0219 | 2 | Variable is assigned but its value is never used | The variable 'x' is assigned but its value is never used |
| Program.cs:9:9:9:11 | Unreachable code detected | CS0162 | 2 | Unreachable code detected | Unreachable code detected |
| Program.cs:9:13:9:13 | Variable is assigned but its value is never used | CS0219 | 2 | Variable is assigned but its value is never used | The variable 'y' is assigned but its value is never used |
metricTests
| compilation | Test passed |
compilation
| compilation | compilations |
compilationArguments
| compilation | 0 | -unsafe |
| compilation | 1 | -target:library |
| compilation | 2 | /noconfig |
| compilation | 3 | /r:System.Private.CoreLib.dll |
| compilation | 4 | /r:System.dll |
| compilation | 5 | /r:System.Core.dll |
| compilation | 6 | /r:System.Runtime.dll |
| compilation | 7 | /r:System.Console.dll |
| compilation | 8 | --console |
| compilation | 9 | --verbosity |
| compilation | 10 | 2 |
| compilation | 11 | Program.cs |
compilationFiles
| compilation | 0 | Program.cs:0:0:0:0 | Program.cs |
compilationFolder
| compilation | folder://compilations | compilations |

View File

@@ -0,0 +1,26 @@
import csharp
import semmle.code.csharp.commons.Diagnostics
query predicate diags(Diagnostic d, string id, int severity, string message, string fullMessage) {
id = d.getId() and
severity = d.getSeverity() and
message = d.getMessage() and
fullMessage = d.getFullMessage()
}
query predicate metricTests(Compilation compilation, string message) {
message = "Test passed" and
forall(int metric | metric in [0 .. 6] | compilation.getMetric(metric) > 0.0)
}
query predicate compilation(Compilation c, string f) { f = c.getDirectoryString() }
query predicate compilationArguments(Compilation compilation, int i, string arg) {
arg = compilation.getArgument(i)
}
query predicate compilationFiles(Compilation compilation, int i, File f) {
f = compilation.getFile(i)
}
query predicate compilationFolder(Compilation c, Folder f) { f = c.getFolder() }

View File

@@ -0,0 +1,11 @@
using System; // CS8019
class Class
{
static void Main(string[] args)
{
var x = 2; // CS0219
return;
var y = 3; // CS0219, CS0162
}
}