hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3136 from calumgrant/cs/buildless-extraction

Browse Source 
C#: Improvements to buildless extraction
This commit is contained in:
Tom Hvitved
2020-04-07 08:52:00 +02:00
committed by GitHub
parent bacb11a563 adde52d33c
commit 6685a5ed4d
26 changed files with 408 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -19,5 +19,6 @@
# It's useful (though not required) to be able to unpack codeql in the ql checkout itself
/codeql/
.vscode/settings.json
csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/launchSettings.json
.vscode

14
csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs
View File

@@ -163,7 +163,19 @@ namespace Semmle.BuildAnalyser
/// </summary>
/// <param name="filepath">The filename to query.</param>
/// <returns>The assembly info.</returns>
public AssemblyInfo GetAssemblyInfo(string filepath) => assemblyInfo[filepath];
public AssemblyInfo GetAssemblyInfo(string filepath)
{
if(assemblyInfo.TryGetValue(filepath, out var info))
{
return info;
}
else
{
info = AssemblyInfo.ReadFromFile(filepath);
assemblyInfo.Add(filepath, info);
return info;
}
}
// List of pending DLLs to index.
readonly List<string> dlls = new List<string>();

249
csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs
View File

@@ -1,10 +1,13 @@
using System;
using Semmle.Util;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using Semmle.Util;
using Semmle.Extraction.CSharp.Standalone;
using System.Threading.Tasks;
using System.Collections.Concurrent;
using System.Text;
using System.Security.Cryptography;
namespace Semmle.BuildAnalyser
{
@@ -43,19 +46,18 @@ namespace Semmle.BuildAnalyser
/// <summary>
/// Main implementation of the build analysis.
/// </summary>
class BuildAnalysis : IBuildAnalysis
class BuildAnalysis : IBuildAnalysis, IDisposable
{
readonly AssemblyCache assemblyCache;
readonly NugetPackages nuget;
readonly IProgressMonitor progressMonitor;
HashSet<string> usedReferences = new HashSet<string>();
readonly HashSet<string> usedSources = new HashSet<string>();
readonly HashSet<string> missingSources = new HashSet<string>();
readonly Dictionary<string, string> unresolvedReferences = new Dictionary<string, string>();
readonly DirectoryInfo sourceDir;
int failedProjects, succeededProjects;
readonly string[] allSources;
int conflictedReferences = 0;
private readonly AssemblyCache assemblyCache;
private readonly NugetPackages nuget;
private readonly IProgressMonitor progressMonitor;
private readonly IDictionary<string, bool> usedReferences = new ConcurrentDictionary<string, bool>();
private readonly IDictionary<string, bool> sources = new ConcurrentDictionary<string, bool>();
private readonly IDictionary<string, string> unresolvedReferences = new ConcurrentDictionary<string, string>();
private readonly DirectoryInfo sourceDir;
private int failedProjects, succeededProjects;
private readonly string[] allSources;
private int conflictedReferences = 0;
/// <summary>
/// Performs a C# build analysis.
@@ -64,6 +66,8 @@ namespace Semmle.BuildAnalyser
/// <param name="progress">Display of analysis progress.</param>
public BuildAnalysis(Options options, IProgressMonitor progress)
{
var startTime = DateTime.Now;
progressMonitor = progress;
sourceDir = new DirectoryInfo(options.SrcDir);
@@ -74,36 +78,43 @@ namespace Semmle.BuildAnalyser
Where(d => !options.ExcludesFile(d)).
ToArray();
var dllDirNames = options.DllDirs.Select(Path.GetFullPath);
var dllDirNames = options.DllDirs.Select(Path.GetFullPath).ToList();
PackageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
if (options.UseNuGet)
{
nuget = new NugetPackages(sourceDir.FullName);
ReadNugetFiles();
dllDirNames = dllDirNames.Concat(Enumerators.Singleton(nuget.PackageDirectory));
try
{
nuget = new NugetPackages(sourceDir.FullName, PackageDirectory);
ReadNugetFiles();
}
catch(FileNotFoundException)
{
progressMonitor.MissingNuGet();
}
}
// Find DLLs in the .Net Framework
if (options.ScanNetFrameworkDlls)
{
dllDirNames = dllDirNames.Concat(Runtime.Runtimes.Take(1));
dllDirNames.Add(Runtime.Runtimes.First());
}
assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress);
// These files can sometimes prevent `dotnet restore` from working correctly.
using (new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories)))
using (new FileRenamer(sourceDir.GetFiles("Directory.Build.props", SearchOption.AllDirectories)))
{
var solutions = options.SolutionFile != null ?
new[] { options.SolutionFile } :
sourceDir.GetFiles("*.sln", SearchOption.AllDirectories).Select(d => d.FullName);
// Analyse all .csproj files in the source tree.
if (options.SolutionFile != null)
{
AnalyseSolution(options.SolutionFile);
}
else if (options.AnalyseCsProjFiles)
{
AnalyseProjectFiles();
}
RestoreSolutions(solutions);
dllDirNames.Add(PackageDirectory.DirInfo.FullName);
assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress);
AnalyseSolutions(solutions);
if (!options.AnalyseCsProjFiles)
{
usedReferences = new HashSet<string>(assemblyCache.AllAssemblies.Select(a => a.Filename));
foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename))
UseReference(filename);
}
ResolveConflicts();
@@ -114,7 +125,7 @@ namespace Semmle.BuildAnalyser
}
// Output the findings
foreach (var r in usedReferences)
foreach (var r in usedReferences.Keys)
{
progressMonitor.ResolvedReference(r);
}
@@ -132,7 +143,27 @@ namespace Semmle.BuildAnalyser
UnresolvedReferences.Count(),
conflictedReferences,
succeededProjects + failedProjects,
failedProjects);
failedProjects,
DateTime.Now - startTime);
}
/// <summary>
/// Computes a unique temp directory for the packages associated
/// with this source tree. Use a SHA1 of the directory name.
/// </summary>
/// <param name="srcDir"></param>
/// <returns>The full path of the temp directory.</returns>
private static string ComputeTempDirectory(string srcDir)
{
var bytes = Encoding.Unicode.GetBytes(srcDir);
using var sha1 = new SHA1CryptoServiceProvider();
var sha = sha1.ComputeHash(bytes);
var sb = new StringBuilder();
foreach (var b in sha.Take(8))
sb.AppendFormat("{0:x2}", b);
return Path.Combine(Path.GetTempPath(), "GitHub", "packages", sb.ToString());
}
/// <summary>
@@ -143,7 +174,7 @@ namespace Semmle.BuildAnalyser
void ResolveConflicts()
{
var sortedReferences = usedReferences.
Select(r => assemblyCache.GetAssemblyInfo(r)).
Select(r => assemblyCache.GetAssemblyInfo(r.Key)).
OrderBy(r => r.Version).
ToArray();
@@ -154,7 +185,9 @@ namespace Semmle.BuildAnalyser
finalAssemblyList[r.Name] = r;
// Update the used references list
usedReferences = new HashSet<string>(finalAssemblyList.Select(r => r.Value.Filename));
usedReferences.Clear();
foreach (var r in finalAssemblyList.Select(r => r.Value.Filename))
UseReference(r);
// Report the results
foreach (var r in sortedReferences)
@@ -183,7 +216,7 @@ namespace Semmle.BuildAnalyser
/// <param name="reference">The filename of the reference.</param>
void UseReference(string reference)
{
usedReferences.Add(reference);
usedReferences[reference] = true;
}
/// <summary>
@@ -192,25 +225,18 @@ namespace Semmle.BuildAnalyser
/// <param name="sourceFile">The source file.</param>
void UseSource(FileInfo sourceFile)
{
if (sourceFile.Exists)
{
usedSources.Add(sourceFile.FullName);
}
else
{
missingSources.Add(sourceFile.FullName);
}
sources[sourceFile.FullName] = sourceFile.Exists;
}
/// <summary>
/// The list of resolved reference files.
/// </summary>
public IEnumerable<string> ReferenceFiles => this.usedReferences;
public IEnumerable<string> ReferenceFiles => this.usedReferences.Keys;
/// <summary>
/// The list of source files used in projects.
/// </summary>
public IEnumerable<string> ProjectSourceFiles => usedSources;
public IEnumerable<string> ProjectSourceFiles => sources.Where(s => s.Value).Select(s => s.Key);
/// <summary>
/// All of the source files in the source directory.
@@ -226,7 +252,7 @@ namespace Semmle.BuildAnalyser
/// List of source files which were mentioned in project files but
/// do not exist on the file system.
/// </summary>
public IEnumerable<string> MissingSourceFiles => missingSources;
public IEnumerable<string> MissingSourceFiles => sources.Where(s => !s.Value).Select(s => s.Key);
/// <summary>
/// Record that a particular reference couldn't be resolved.
@@ -239,74 +265,101 @@ namespace Semmle.BuildAnalyser
unresolvedReferences[id] = projectFile;
}
/// <summary>
/// Performs an analysis of all .csproj files.
/// </summary>
void AnalyseProjectFiles()
{
AnalyseProjectFiles(sourceDir.GetFiles("*.csproj", SearchOption.AllDirectories));
}
readonly TemporaryDirectory PackageDirectory;
/// <summary>
/// Reads all the source files and references from the given list of projects.
/// </summary>
/// <param name="projectFiles">The list of projects to analyse.</param>
void AnalyseProjectFiles(FileInfo[] projectFiles)
void AnalyseProjectFiles(IEnumerable<FileInfo> projectFiles)
{
progressMonitor.AnalysingProjectFiles(projectFiles.Count());
foreach (var proj in projectFiles)
AnalyseProject(proj);
}
void AnalyseProject(FileInfo project)
{
if(!project.Exists)
{
try
{
var csProj = new CsProjFile(proj);
progressMonitor.MissingProject(project.FullName);
return;
}
foreach (var @ref in csProj.References)
{
AssemblyInfo resolved = assemblyCache.ResolveReference(@ref);
if (!resolved.Valid)
{
UnresolvedReference(@ref, proj.FullName);
}
else
{
UseReference(resolved.Filename);
}
}
try
{
var csProj = new CsProjFile(project);
foreach (var src in csProj.Sources)
{
// Make a note of which source files the projects use.
// This information doesn't affect the build but is dumped
// as diagnostic output.
UseSource(new FileInfo(src));
}
++succeededProjects;
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
foreach (var @ref in csProj.References)
{
++failedProjects;
progressMonitor.FailedProjectFile(proj.FullName, ex.Message);
AssemblyInfo resolved = assemblyCache.ResolveReference(@ref);
if (!resolved.Valid)
{
UnresolvedReference(@ref, project.FullName);
}
else
{
UseReference(resolved.Filename);
}
}
foreach (var src in csProj.Sources)
{
// Make a note of which source files the projects use.
// This information doesn't affect the build but is dumped
// as diagnostic output.
UseSource(new FileInfo(src));
}
++succeededProjects;
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
++failedProjects;
progressMonitor.FailedProjectFile(project.FullName, ex.Message);
}
}
void Restore(string projectOrSolution)
{
int exit = DotNet.RestoreToDirectory(projectOrSolution, PackageDirectory.DirInfo.FullName);
switch(exit)
{
case 0:
case 1:
// No errors
break;
default:
progressMonitor.CommandFailed("dotnet", $"restore \"{projectOrSolution}\"", exit);
break;
}
}
/// <summary>
/// Delete packages directory.
/// </summary>
public void Cleanup()
public void RestoreSolutions(IEnumerable<string> solutions)
{
if (nuget != null) nuget.Cleanup(progressMonitor);
Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = 4 }, Restore);
}
/// <summary>
/// Analyse all project files in a given solution only.
/// </summary>
/// <param name="solutionFile">The filename of the solution.</param>
public void AnalyseSolution(string solutionFile)
public void AnalyseSolutions(IEnumerable<string> solutions)
{
var sln = new SolutionFile(solutionFile);
AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).ToArray());
Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = 4 } , solutionFile =>
{
try
{
var sln = new SolutionFile(solutionFile);
progressMonitor.AnalysingSolution(solutionFile);
AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).Where(p => p.Exists));
}
catch (Microsoft.Build.Exceptions.InvalidProjectFileException ex)
{
progressMonitor.FailedProjectFile(solutionFile, ex.BaseMessage);
}
});
}
public void Dispose()
{
PackageDirectory?.Dispose();
}
}
}

64
csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs
View File

@@ -10,12 +10,18 @@ namespace Semmle.BuildAnalyser
/// </summary>
class CsProjFile
{
private string Filename { get; }
private string Directory => Path.GetDirectoryName(Filename);
/// <summary>
/// Reads the .csproj file.
/// </summary>
/// <param name="filename">The .csproj file.</param>
public CsProjFile(FileInfo filename)
{
Filename = filename.FullName;
try
{
// This can fail if the .csproj is invalid or has
@@ -39,7 +45,7 @@ namespace Semmle.BuildAnalyser
/// and there seems to be no way to make it succeed. Fails on Linux.
/// </summary>
/// <param name="filename">The file to read.</param>
public void ReadMsBuildProject(FileInfo filename)
private void ReadMsBuildProject(FileInfo filename)
{
var msbuildProject = new Microsoft.Build.Execution.ProjectInstance(filename.FullName);
@@ -62,7 +68,7 @@ namespace Semmle.BuildAnalyser
/// fallback if ReadMsBuildProject() fails.
/// </summary>
/// <param name="filename">The .csproj file.</param>
public void ReadProjectFileAsXml(FileInfo filename)
private void ReadProjectFileAsXml(FileInfo filename)
{
var projFile = new XmlDocument();
var mgr = new XmlNamespaceManager(projFile.NameTable);
@@ -71,22 +77,48 @@ namespace Semmle.BuildAnalyser
var projDir = filename.Directory;
var root = projFile.DocumentElement;
references =
root.SelectNodes("/msbuild:Project/msbuild:ItemGroup/msbuild:Reference/@Include", mgr).
NodeList().
Select(node => node.Value).
ToArray();
// Figure out if it's dotnet core
var relativeCsIncludes =
root.SelectNodes("/msbuild:Project/msbuild:ItemGroup/msbuild:Compile/@Include", mgr).
NodeList().
Select(node => node.Value).
ToArray();
bool netCoreProjectFile = root.GetAttribute("Sdk") == "Microsoft.NET.Sdk";
csFiles = relativeCsIncludes.
Select(cs => Path.DirectorySeparatorChar == '/' ? cs.Replace("\\", "/") : cs).
Select(f => Path.GetFullPath(Path.Combine(projDir.FullName, f))).
ToArray();
if (netCoreProjectFile)
{
var relativeCsIncludes =
root.SelectNodes("/Project/ItemGroup/Compile/@Include", mgr).
NodeList().
Select(node => node.Value).
ToArray();
var explicitCsFiles = relativeCsIncludes.
Select(cs => Path.DirectorySeparatorChar == '/' ? cs.Replace("\\", "/") : cs).
Select(f => Path.GetFullPath(Path.Combine(projDir.FullName, f)));
var additionalCsFiles = System.IO.Directory.GetFiles(Directory, "*.cs", SearchOption.AllDirectories);
csFiles = explicitCsFiles.Concat(additionalCsFiles).ToArray();
references = new string[0];
}
else
{
references =
root.SelectNodes("/msbuild:Project/msbuild:ItemGroup/msbuild:Reference/@Include", mgr).
NodeList().
Select(node => node.Value).
ToArray();
var relativeCsIncludes =
root.SelectNodes("/msbuild:Project/msbuild:ItemGroup/msbuild:Compile/@Include", mgr).
NodeList().
Select(node => node.Value).
ToArray();
csFiles = relativeCsIncludes.
Select(cs => Path.DirectorySeparatorChar == '/' ? cs.Replace("\\", "/") : cs).
Select(f => Path.GetFullPath(Path.Combine(projDir.FullName, f))).
ToArray();
}
}
string[] references;

21
csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs Normal file
View File

@@ -0,0 +1,21 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
namespace Semmle.BuildAnalyser
{
/// <summary>
/// Utilities to run the "dotnet" command.
/// </summary>
static class DotNet
{
public static int RestoreToDirectory(string projectOrSolutionFile, string packageDirectory)
{
using var proc = Process.Start("dotnet", $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true");
proc.WaitForExit();
return proc.ExitCode;
}
}
}

85
csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs
View File

@@ -1,10 +1,9 @@
using System;
using Semmle.Util;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
namespace Semmle.BuildAnalyser
{
@@ -19,10 +18,10 @@ namespace Semmle.BuildAnalyser
/// Create the package manager for a specified source tree.
/// </summary>
/// <param name="sourceDir">The source directory.</param>
public NugetPackages(string sourceDir)
public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory)
{
SourceDirectory = sourceDir;
PackageDirectory = computeTempDirectory(sourceDir);
PackageDirectory = packageDirectory;
// Expect nuget.exe to be in a `nuget` directory under the directory containing this exe.
var currentAssembly = System.Reflection.Assembly.GetExecutingAssembly().Location;
@@ -50,45 +49,12 @@ namespace Semmle.BuildAnalyser
/// </summary>
public IEnumerable<FileInfo> PackageFiles => packages;
// Whether to delete the packages directory prior to each run.
// Makes each build more reproducible.
const bool cleanupPackages = true;
public void Cleanup(IProgressMonitor pm)
{
var packagesDirectory = new DirectoryInfo(PackageDirectory);
if (packagesDirectory.Exists)
{
try
{
packagesDirectory.Delete(true);
}
catch (System.IO.IOException ex)
{
pm.Warning(string.Format("Couldn't delete package directory - it's probably held open by something else: {0}", ex.Message));
}
}
}
/// <summary>
/// Download the packages to the temp folder.
/// </summary>
/// <param name="pm">The progress monitor used for reporting errors etc.</param>
public void InstallPackages(IProgressMonitor pm)
{
if (cleanupPackages)
{
Cleanup(pm);
}
var packagesDirectory = new DirectoryInfo(PackageDirectory);
if (!Directory.Exists(PackageDirectory))
{
packagesDirectory.Create();
}
foreach (var package in packages)
{
RestoreNugetPackage(package.FullName, pm);
@@ -109,31 +75,7 @@ namespace Semmle.BuildAnalyser
/// This will be in the Temp location
/// so as to not trample the source tree.
/// </summary>
public string PackageDirectory
{
get;
private set;
}
readonly SHA1CryptoServiceProvider sha1 = new SHA1CryptoServiceProvider();
/// <summary>
/// Computes a unique temp directory for the packages associated
/// with this source tree. Use a SHA1 of the directory name.
/// </summary>
/// <param name="srcDir"></param>
/// <returns>The full path of the temp directory.</returns>
string computeTempDirectory(string srcDir)
{
var bytes = Encoding.Unicode.GetBytes(srcDir);
var sha = sha1.ComputeHash(bytes);
var sb = new StringBuilder();
foreach (var b in sha.Take(8))
sb.AppendFormat("{0:x2}", b);
return Path.Combine(Path.GetTempPath(), "Semmle", "packages", sb.ToString());
}
public TemporaryDirectory PackageDirectory { get; }
/// <summary>
/// Restore all files in a specified package.
@@ -171,16 +113,15 @@ namespace Semmle.BuildAnalyser
try
{
using (var p = Process.Start(pi))
{
string output = p.StandardOutput.ReadToEnd();
string error = p.StandardError.ReadToEnd();
using var p = Process.Start(pi);
p.WaitForExit();
if (p.ExitCode != 0)
{
pm.FailedNugetCommand(pi.FileName, pi.Arguments, output + error);
}
string output = p.StandardOutput.ReadToEnd();
string error = p.StandardError.ReadToEnd();
p.WaitForExit();
if (p.ExitCode != 0)
{
pm.FailedNugetCommand(pi.FileName, pi.Arguments, output + error);
}
}
catch (Exception ex)

19
csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs
View File

@@ -23,7 +23,7 @@ namespace Semmle.Extraction.CSharp.Standalone
/// <summary>
/// Searches for source/references and creates separate extractions.
/// </summary>
class Analysis
class Analysis : IDisposable
{
readonly ILogger logger;
@@ -71,12 +71,9 @@ namespace Semmle.Extraction.CSharp.Standalone
projectExtraction.Sources.AddRange(options.SolutionFile == null ? buildAnalysis.AllSourceFiles : buildAnalysis.ProjectSourceFiles);
}
/// <summary>
/// Delete any Nuget assemblies.
/// </summary>
public void Cleanup()
public void Dispose()
{
buildAnalysis.Cleanup();
buildAnalysis.Dispose();
}
};
@@ -85,8 +82,9 @@ namespace Semmle.Extraction.CSharp.Standalone
static int Main(string[] args)
{
var options = Options.Create(args);
// options.CIL = true; // To do: Enable this
var output = new ConsoleLogger(options.Verbosity);
var a = new Analysis(output);
using var a = new Analysis(output);
if (options.Help)
{
@@ -97,6 +95,8 @@ namespace Semmle.Extraction.CSharp.Standalone
if (options.Errors)
return 1;
var start = DateTime.Now;
output.Log(Severity.Info, "Running C# standalone extractor");
a.AnalyseProjects(options);
int sourceFiles = a.Extraction.Sources.Count();
@@ -117,10 +117,9 @@ namespace Semmle.Extraction.CSharp.Standalone
new ExtractionProgress(output),
new FileLogger(options.Verbosity, Extractor.GetCSharpLogPath()),
options);
output.Log(Severity.Info, "Extraction complete");
output.Log(Severity.Info, $"Extraction completed in {DateTime.Now-start}");
}
a.Cleanup();
return 0;
}
@@ -151,7 +150,7 @@ namespace Semmle.Extraction.CSharp.Standalone
public void MissingSummary(int missingTypes, int missingNamespaces)
{
logger.Log(Severity.Info, "Failed to resolve {0} types and {1} namespaces", missingTypes, missingNamespaces);
logger.Log(Severity.Info, "Failed to resolve {0} types in {1} namespaces", missingTypes, missingNamespaces);
}
}
}

28
csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs
View File

@@ -1,4 +1,5 @@
using Semmle.Util.Logging;
using System;
namespace Semmle.BuildAnalyser
{
@@ -9,15 +10,17 @@ namespace Semmle.BuildAnalyser
{
void FindingFiles(string dir);
void UnresolvedReference(string id, string project);
void AnalysingProjectFiles(int count);
void AnalysingSolution(string filename);
void FailedProjectFile(string filename, string reason);
void FailedNugetCommand(string exe, string args, string message);
void NugetInstall(string package);
void ResolvedReference(string filename);
void Summary(int existingSources, int usedSources, int missingSources, int references, int unresolvedReferences, int resolvedConflicts, int totalProjects, int failedProjects);
void Summary(int existingSources, int usedSources, int missingSources, int references, int unresolvedReferences, int resolvedConflicts, int totalProjects, int failedProjects, TimeSpan analysisTime);
void Warning(string message);
void ResolvedConflict(string asm1, string asm2);
void MissingProject(string projectFile);
void CommandFailed(string exe, string arguments, int exitCode);
void MissingNuGet();
}
class ProgressMonitor : IProgressMonitor
@@ -46,9 +49,9 @@ namespace Semmle.BuildAnalyser
logger.Log(Severity.Debug, "Unresolved {0} referenced by {1}", id, project);
}
public void AnalysingProjectFiles(int count)
public void AnalysingSolution(string filename)
{
logger.Log(Severity.Info, "Analyzing project files...");
logger.Log(Severity.Info, $"Analyzing {filename}...");
}
public void FailedProjectFile(string filename, string reason)
@@ -73,7 +76,9 @@ namespace Semmle.BuildAnalyser
}
public void Summary(int existingSources, int usedSources, int missingSources,
int references, int unresolvedReferences, int resolvedConflicts, int totalProjects, int failedProjects)
int references, int unresolvedReferences,
int resolvedConflicts, int totalProjects, int failedProjects,
TimeSpan analysisTime)
{
logger.Log(Severity.Info, "");
logger.Log(Severity.Info, "Build analysis summary:");
@@ -85,6 +90,7 @@ namespace Semmle.BuildAnalyser
logger.Log(Severity.Info, "{0, 6} resolved assembly conflicts", resolvedConflicts);
logger.Log(Severity.Info, "{0, 6} projects", totalProjects);
logger.Log(Severity.Info, "{0, 6} missing/failed projects", failedProjects);
logger.Log(Severity.Info, "Build analysis completed in {0}", analysisTime);
}
public void Warning(string message)
@@ -94,12 +100,22 @@ namespace Semmle.BuildAnalyser
public void ResolvedConflict(string asm1, string asm2)
{
logger.Log(Severity.Info, "Resolved {0} as {1}", asm1, asm2);
logger.Log(Severity.Debug, "Resolved {0} as {1}", asm1, asm2);
}
public void MissingProject(string projectFile)
{
logger.Log(Severity.Info, "Solution is missing {0}", projectFile);
}
public void CommandFailed(string exe, string arguments, int exitCode)
{
logger.Log(Severity.Error, $"Command {exe} {arguments} failed with exit code {exitCode}");
}
public void MissingNuGet()
{
logger.Log(Severity.Error, "Missing nuget.exe");
}
}
}

3
csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj
View File

@@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
@@ -14,6 +14,7 @@
<ItemGroup>
<ProjectReference Include="..\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj" />
<ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
</ItemGroup>
<ItemGroup>

6
csharp/extractor/Semmle.Extraction.CSharp.Standalone/SolutionFile.cs
View File

@@ -12,6 +12,8 @@ namespace Semmle.BuildAnalyser
{
readonly Microsoft.Build.Construction.SolutionFile solutionFile;
private string FullPath { get; }
/// <summary>
/// Read the file.
/// </summary>
@@ -19,8 +21,8 @@ namespace Semmle.BuildAnalyser
public SolutionFile(string filename)
{
// SolutionFile.Parse() expects a rooted path.
var fullPath = Path.GetFullPath(filename);
solutionFile = Microsoft.Build.Construction.SolutionFile.Parse(fullPath);
FullPath = Path.GetFullPath(filename);
solutionFile = Microsoft.Build.Construction.SolutionFile.Parse(FullPath);
}
/// <summary>

5
csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Access.cs
View File

@@ -45,7 +45,10 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
Access(ExpressionNodeInfo info, ISymbol symbol, bool implicitThis, IEntity target)
: base(info.SetKind(AccessKind(info.Context, symbol)))
{
cx.TrapWriter.Writer.expr_access(this, target);
if (!(target is null))
{
cx.TrapWriter.Writer.expr_access(this, target);
}
if (implicitThis && !symbol.IsStatic)
{

4
csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MemberAccess.cs
View File

@@ -71,7 +71,9 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
if (symbol == null)
{
info.Context.ModelError(info.Node, "Failed to determine symbol for member access");
return new MemberAccess(info.SetKind(ExprKind.UNKNOWN), expression, symbol);
// Default to property access - this can still give useful results but
// the target of the expression should be checked in QL.
return new MemberAccess(info.SetKind(ExprKind.PROPERTY_ACCESS), expression, symbol);
}
ExprKind kind;

3
csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs
View File

@@ -23,7 +23,8 @@ namespace Semmle.Extraction.CSharp.Entities
protected override void Populate(TextWriter trapFile)
{
var ns = Namespace.Create(cx, (INamespaceSymbol)cx.GetModel(Node).GetSymbolInfo(Node.Name).Symbol);
var @namespace = (INamespaceSymbol) cx.GetModel(Node).GetSymbolInfo(Node.Name).Symbol;
var ns = Namespace.Create(cx, @namespace);
trapFile.namespace_declarations(this, ns);
trapFile.namespace_declaration_location(this, cx.Create(Node.Name.GetLocation()));

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs
View File

@@ -25,7 +25,7 @@ namespace Semmle.Extraction.CSharp.Entities
{
if (symbol.TypeKind == TypeKind.Error)
{
Context.Extractor.MissingType(symbol.ToString());
Context.Extractor.MissingType(symbol.ToString(), Context.FromSource);
return;
}

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs
View File

@@ -32,7 +32,7 @@ namespace Semmle.Extraction.CSharp.Entities
if (namespaceSymbol == null)
{
cx.Extractor.MissingNamespace(Node.Name.ToFullString());
cx.Extractor.MissingNamespace(Node.Name.ToFullString(), cx.FromSource);
cx.ModelError(Node, "Namespace not found");
return;
}

1
csharp/extractor/Semmle.Extraction.CSharp/SymbolExtensions.cs
View File

@@ -214,7 +214,6 @@ namespace Semmle.Extraction.CSharp
static void BuildNamedTypeId(this INamedTypeSymbol named, Context cx, TextWriter trapFile, Action<Context, TextWriter, ITypeSymbol> subTermAction)
{
bool prefixAssembly = true;
if (cx.Extractor.Standalone) prefixAssembly = false;
if (named.ContainingAssembly is null) prefixAssembly = false;
if (named.IsTupleType)

4
csharp/extractor/Semmle.Extraction/Context.cs
View File

@@ -155,7 +155,7 @@ namespace Semmle.Extraction
#if DEBUG_LABELS
using (var id = new StringWriter())
{
entity.WriteId(id);
entity.WriteQuotedId(id);
CheckEntityHasUniqueLabel(id.ToString(), entity);
}
#endif
@@ -270,6 +270,8 @@ namespace Semmle.Extraction
TrapWriter = trapWriter;
}
public bool FromSource => Scope.FromSource;
public bool IsGlobalContext => Scope.IsGlobalScope;
public readonly ICommentGenerator CommentGenerator = new CommentProcessor();

6
csharp/extractor/Semmle.Extraction/ExtractionScope.cs
View File

@@ -25,6 +25,8 @@ namespace Semmle.Extraction
bool InFileScope(string path);
bool IsGlobalScope { get; }
bool FromSource { get; }
}
/// <summary>
@@ -49,6 +51,8 @@ namespace Semmle.Extraction
public bool InScope(ISymbol symbol) =>
SymbolEqualityComparer.Default.Equals(symbol.ContainingAssembly, assembly) ||
SymbolEqualityComparer.Default.Equals(symbol, assembly);
public bool FromSource => false;
}
/// <summary>
@@ -68,5 +72,7 @@ namespace Semmle.Extraction
public bool InFileScope(string path) => path == sourceTree.FilePath;
public bool InScope(ISymbol symbol) => symbol.Locations.Any(loc => loc.SourceTree == sourceTree);
public bool FromSource => true;
}
}

24
csharp/extractor/Semmle.Extraction/Extractor.cs
View File

@@ -50,13 +50,15 @@ namespace Semmle.Extraction
/// Record a new error type.
/// </summary>
/// <param name="fqn">The display name of the type, qualified where possible.</param>
void MissingType(string fqn);
/// <param name="fromSource">If the missing type was referenced from a source file.</param>
void MissingType(string fqn, bool fromSource);
/// <summary>
/// Record an unresolved `using namespace` directive.
/// </summary>
/// <param name="fqn">The full name of the namespace.</param>
void MissingNamespace(string fqn);
/// <param name="fromSource">If the missing namespace was referenced from a source file.</param>
void MissingNamespace(string fqn, bool fromSource);
/// <summary>
/// The list of missing types.
@@ -167,16 +169,22 @@ namespace Semmle.Extraction
readonly ISet<string> missingTypes = new SortedSet<string>();
readonly ISet<string> missingNamespaces = new SortedSet<string>();
public void MissingType(string fqn)
public void MissingType(string fqn, bool fromSource)
{
lock (mutex)
missingTypes.Add(fqn);
if (fromSource)
{
lock (mutex)
missingTypes.Add(fqn);
}
}
public void MissingNamespace(string fqdn)
public void MissingNamespace(string fqdn, bool fromSource)
{
lock (mutex)
missingNamespaces.Add(fqdn);
if (fromSource)
{
lock (mutex)
missingNamespaces.Add(fqdn);
}
}
public Context CreateContext(Compilation c, TrapWriter trapWriter, IExtractionScope scope)

34
csharp/extractor/Semmle.Util/FileRenamer.cs Normal file
View File

@@ -0,0 +1,34 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace Semmle.Util
{
/// <summary>
/// Utility to temporarily rename a set of files.
/// </summary>
public sealed class FileRenamer : IDisposable
{
readonly string[] files;
const string suffix = ".codeqlhidden";
public FileRenamer(IEnumerable<FileInfo> oldFiles)
{
files = oldFiles.Select(f => f.FullName).ToArray();
foreach (var file in files)
{
File.Move(file, file + suffix);
}
}
public void Dispose()
{
foreach (var file in files)
{
File.Move(file + suffix, file);
}
}
}
}

30
csharp/extractor/Semmle.Util/TemporaryDirectory.cs Normal file
View File

@@ -0,0 +1,30 @@
using System;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
namespace Semmle.Util
{
/// <summary>
/// A temporary directory that is created within the system temp directory.
/// When this object is disposed, the directory is deleted.
/// </summary>
public sealed class TemporaryDirectory : IDisposable
{
public DirectoryInfo DirInfo { get; }
public TemporaryDirectory(string name)
{
DirInfo = new DirectoryInfo(name);
DirInfo.Create();
}
public void Dispose()
{
DirInfo.Delete(true);
}
public override string ToString() => DirInfo.FullName.ToString();
}
}

7
csharp/ql/src/semmle/code/csharp/exprs/Access.qll
View File

@@ -388,7 +388,12 @@ library class PropertyAccessExpr extends Expr, @property_access_expr {
/** Gets the target of this property access. */
Property getProperty() { expr_access(this, result) }
override string toString() { result = "access to property " + this.getProperty().getName() }
override string toString() {
result = "access to property " + this.getProperty().getName()
or
not exists(this.getProperty()) and
result = "access to property (unknown)"
}
}
/**

17
csharp/ql/test/library-tests/standalone/controlflow/cfg.expected
View File

@@ -7,9 +7,14 @@
| ControlFlow.cs:10:9:10:43 | Call (unknown target) | ControlFlow.cs:12:9:12:87 | ...; |
| ControlFlow.cs:10:9:10:43 | call to method | ControlFlow.cs:12:9:12:87 | ...; |
| ControlFlow.cs:10:9:10:44 | ...; | ControlFlow.cs:10:9:10:13 | Expression |
| ControlFlow.cs:10:22:10:22 | access to local variable v | ControlFlow.cs:10:22:10:24 | Expression |
| ControlFlow.cs:10:22:10:24 | Expression | ControlFlow.cs:10:22:10:26 | Expression |
| ControlFlow.cs:10:22:10:26 | Expression | ControlFlow.cs:10:29:10:42 | "This is true" |
| ControlFlow.cs:10:22:10:22 | access to local variable v | ControlFlow.cs:10:22:10:24 | Call (unknown target) |
| ControlFlow.cs:10:22:10:22 | access to local variable v | ControlFlow.cs:10:22:10:24 | access to property (unknown) |
| ControlFlow.cs:10:22:10:24 | Call (unknown target) | ControlFlow.cs:10:22:10:26 | Call (unknown target) |
| ControlFlow.cs:10:22:10:24 | Call (unknown target) | ControlFlow.cs:10:22:10:26 | access to property (unknown) |
| ControlFlow.cs:10:22:10:24 | access to property (unknown) | ControlFlow.cs:10:22:10:26 | Call (unknown target) |
| ControlFlow.cs:10:22:10:24 | access to property (unknown) | ControlFlow.cs:10:22:10:26 | access to property (unknown) |
| ControlFlow.cs:10:22:10:26 | Call (unknown target) | ControlFlow.cs:10:29:10:42 | "This is true" |
| ControlFlow.cs:10:22:10:26 | access to property (unknown) | ControlFlow.cs:10:29:10:42 | "This is true" |
| ControlFlow.cs:10:29:10:42 | "This is true" | ControlFlow.cs:10:9:10:43 | Call (unknown target) |
| ControlFlow.cs:10:29:10:42 | "This is true" | ControlFlow.cs:10:9:10:43 | call to method |
| ControlFlow.cs:12:9:12:86 | Call (unknown target) | ControlFlow.cs:12:37:12:47 | Expression |
@@ -20,5 +25,7 @@
| ControlFlow.cs:12:51:12:62 | access to field Empty | ControlFlow.cs:12:37:12:62 | ... = ... |
| ControlFlow.cs:12:65:12:75 | Expression | ControlFlow.cs:12:79:12:79 | access to local variable v |
| ControlFlow.cs:12:65:12:84 | ... = ... | ControlFlow.cs:12:35:12:86 | { ..., ... } |
| ControlFlow.cs:12:79:12:79 | access to local variable v | ControlFlow.cs:12:79:12:84 | Expression |
| ControlFlow.cs:12:79:12:84 | Expression | ControlFlow.cs:12:65:12:84 | ... = ... |
| ControlFlow.cs:12:79:12:79 | access to local variable v | ControlFlow.cs:12:79:12:84 | Call (unknown target) |
| ControlFlow.cs:12:79:12:79 | access to local variable v | ControlFlow.cs:12:79:12:84 | access to property (unknown) |
| ControlFlow.cs:12:79:12:84 | Call (unknown target) | ControlFlow.cs:12:65:12:84 | ... = ... |
| ControlFlow.cs:12:79:12:84 | access to property (unknown) | ControlFlow.cs:12:65:12:84 | ... = ... |

2
csharp/ql/test/library-tests/standalone/errorrecovery/ErrorCalls.expected
View File

@@ -2,5 +2,5 @@
| errors.cs:43:21:43:28 | errors.cs:43:21:43:28 | object creation of type C1 | C1 |
| errors.cs:44:13:44:19 | errors.cs:44:13:44:19 | call to method m1 | m1 |
| errors.cs:45:13:45:19 | errors.cs:45:13:45:19 | call to method m2 | m2 |
| errors.cs:46:13:46:38 | errors.cs:46:13:46:38 | call to method | none |
| errors.cs:46:13:46:38 | errors.cs:46:13:46:38 | call to method WriteLine | WriteLine |
| errors.cs:53:17:53:25 | errors.cs:53:17:53:25 | object creation of type C2 | none |

5
csharp/ql/test/library-tests/standalone/regressions/ConstCase.expected
View File

@@ -1,2 +1,3 @@
| regressions.cs:16:13:16:37 | case ...: | regressions.cs:16:18:16:36 | Expression |
| regressions.cs:18:13:18:37 | case ...: | regressions.cs:18:18:18:36 | Expression |
| regressions.cs:16:13:16:37 | case ...: | regressions.cs:16:18:16:36 | access to property (unknown) |
| regressions.cs:18:13:18:37 | case ...: | regressions.cs:18:18:18:36 | access to property (unknown) |
| regressions.cs:20:13:20:23 | case ...: | regressions.cs:20:18:20:22 | Int32 x |

4
csharp/ql/test/library-tests/standalone/regressions/ConstCase.ql
View File

@@ -1,7 +1,5 @@
import csharp
from Case c, Expr e
where
e = c.getPattern().stripCasts() and
(e instanceof @unknown_expr or e instanceof ConstantPatternExpr)
where e = c.getPattern().stripCasts()
select c, e