diff --git a/.gitignore b/.gitignore index 1768ebb161f..c0e9ed803ff 100644 --- a/.gitignore +++ b/.gitignore @@ -16,5 +16,6 @@ # It's useful (though not required) to be able to unpack codeql in the ql checkout itself /codeql/ -.vscode/settings.json + csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/launchSettings.json +.vscode diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs index db2664bf4c9..f93911b8a38 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs @@ -163,7 +163,19 @@ namespace Semmle.BuildAnalyser /// /// The filename to query. /// The assembly info. - public AssemblyInfo GetAssemblyInfo(string filepath) => assemblyInfo[filepath]; + public AssemblyInfo GetAssemblyInfo(string filepath) + { + if(assemblyInfo.TryGetValue(filepath, out var info)) + { + return info; + } + else + { + info = AssemblyInfo.ReadFromFile(filepath); + assemblyInfo.Add(filepath, info); + return info; + } + } // List of pending DLLs to index. readonly List dlls = new List(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs index b0ef328bcd0..dfc110c0d03 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs @@ -5,6 +5,8 @@ using System.Linq; using System.Runtime.InteropServices; using Semmle.Util; using Semmle.Extraction.CSharp.Standalone; +using System.Threading.Tasks; +using System.Collections.Concurrent; namespace Semmle.BuildAnalyser { @@ -56,6 +58,7 @@ namespace Semmle.BuildAnalyser int failedProjects, succeededProjects; readonly string[] allSources; int conflictedReferences = 0; + object mutex = new object(); /// /// Performs a C# build analysis. @@ -64,6 +67,8 @@ namespace Semmle.BuildAnalyser /// Display of analysis progress. public BuildAnalysis(Options options, IProgressMonitor progress) { + var startTime = DateTime.Now; + progressMonitor = progress; sourceDir = new DirectoryInfo(options.SrcDir); @@ -74,31 +79,43 @@ namespace Semmle.BuildAnalyser Where(d => !options.ExcludesFile(d)). ToArray(); - var dllDirNames = options.DllDirs.Select(Path.GetFullPath); + var dllDirNames = options.DllDirs.Select(Path.GetFullPath).ToList(); + PackageDirectory = TemporaryDirectory.CreateTempDirectory(sourceDir.FullName, progressMonitor); if (options.UseNuGet) { - nuget = new NugetPackages(sourceDir.FullName); - ReadNugetFiles(); - dllDirNames = dllDirNames.Concat(Enumerators.Singleton(nuget.PackageDirectory)); + try + { + nuget = new NugetPackages(sourceDir.FullName, PackageDirectory); + ReadNugetFiles(); + } + catch(FileNotFoundException) + { + progressMonitor.MissingNuGet(); + } } // Find DLLs in the .Net Framework if (options.ScanNetFrameworkDlls) { - dllDirNames = dllDirNames.Concat(Runtime.Runtimes.Take(1)); + dllDirNames.Add(Runtime.Runtimes.First()); } - - assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress); - - // Analyse all .csproj files in the source tree. - if (options.SolutionFile != null) + { - AnalyseSolution(options.SolutionFile); - } - else if (options.AnalyseCsProjFiles) - { - AnalyseProjectFiles(); + using var renamer1 = new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories)); + using var renamer2 = new FileRenamer(sourceDir.GetFiles("Directory.Build.props", SearchOption.AllDirectories)); + + var solutions = options.SolutionFile != null ? + new[] { options.SolutionFile } : + sourceDir.GetFiles("*.sln", SearchOption.AllDirectories).Select(d => d.FullName); + + + RestoreSolutions(solutions); + dllDirNames.Add(PackageDirectory.DirInfo.FullName); + assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress); + AnalyseSolutions(solutions); + + usedReferences = new HashSet(assemblyCache.AllAssemblies.Select(a => a.Filename)); } if (!options.AnalyseCsProjFiles) @@ -106,6 +123,7 @@ namespace Semmle.BuildAnalyser usedReferences = new HashSet(assemblyCache.AllAssemblies.Select(a => a.Filename)); } + ResolveConflicts(); if (options.UseMscorlib) @@ -133,6 +151,8 @@ namespace Semmle.BuildAnalyser conflictedReferences, succeededProjects + failedProjects, failedProjects); + + Console.WriteLine($"Build analysis completed in {DateTime.Now - startTime}"); } /// @@ -183,7 +203,8 @@ namespace Semmle.BuildAnalyser /// The filename of the reference. void UseReference(string reference) { - usedReferences.Add(reference); + lock (mutex) + usedReferences.Add(reference); } /// @@ -194,11 +215,13 @@ namespace Semmle.BuildAnalyser { if (sourceFile.Exists) { - usedSources.Add(sourceFile.FullName); + lock(mutex) + usedSources.Add(sourceFile.FullName); } else { - missingSources.Add(sourceFile.FullName); + lock(mutex) + missingSources.Add(sourceFile.FullName); } } @@ -236,59 +259,63 @@ namespace Semmle.BuildAnalyser /// The project file making the reference. void UnresolvedReference(string id, string projectFile) { - unresolvedReferences[id] = projectFile; + lock(mutex) + unresolvedReferences[id] = projectFile; } - /// - /// Performs an analysis of all .csproj files. - /// - void AnalyseProjectFiles() - { - AnalyseProjectFiles(sourceDir.GetFiles("*.csproj", SearchOption.AllDirectories)); - } + TemporaryDirectory PackageDirectory; /// /// Reads all the source files and references from the given list of projects. /// /// The list of projects to analyse. - void AnalyseProjectFiles(FileInfo[] projectFiles) + void AnalyseProjectFiles(IEnumerable projectFiles) { - progressMonitor.AnalysingProjectFiles(projectFiles.Count()); - foreach (var proj in projectFiles) + AnalyseProject(proj); + } + + void AnalyseProject(FileInfo project) + { + if(!project.Exists) { - try - { - var csProj = new CsProjFile(proj); - - foreach (var @ref in csProj.References) - { - AssemblyInfo resolved = assemblyCache.ResolveReference(@ref); - if (!resolved.Valid) - { - UnresolvedReference(@ref, proj.FullName); - } - else - { - UseReference(resolved.Filename); - } - } - - foreach (var src in csProj.Sources) - { - // Make a note of which source files the projects use. - // This information doesn't affect the build but is dumped - // as diagnostic output. - UseSource(new FileInfo(src)); - } - ++succeededProjects; - } - catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] - { - ++failedProjects; - progressMonitor.FailedProjectFile(proj.FullName, ex.Message); - } + progressMonitor.MissingProject(project.FullName); + return; } + + try + { + var csProj = new CsProjFile(project); + + foreach (var @ref in csProj.References) + { + AssemblyInfo resolved = assemblyCache.ResolveReference(@ref); + if (!resolved.Valid) + { + UnresolvedReference(@ref, project.FullName); + } + else + { + UseReference(resolved.Filename); + } + } + + foreach (var src in csProj.Sources) + { + // Make a note of which source files the projects use. + // This information doesn't affect the build but is dumped + // as diagnostic output. + UseSource(new FileInfo(src)); + } + + ++succeededProjects; + } + catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] + { + ++failedProjects; + progressMonitor.FailedProjectFile(project.FullName, ex.Message); + } + } /// @@ -296,17 +323,36 @@ namespace Semmle.BuildAnalyser /// public void Cleanup() { - if (nuget != null) nuget.Cleanup(progressMonitor); + PackageDirectory?.Cleanup(); } - /// - /// Analyse all project files in a given solution only. - /// - /// The filename of the solution. - public void AnalyseSolution(string solutionFile) + void Restore(string projectOrSolution) { - var sln = new SolutionFile(solutionFile); - AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).ToArray()); + int exit = DotNet.RestoreToDirectory(projectOrSolution, PackageDirectory.DirInfo.FullName); + if (exit != 0) + progressMonitor.CommandFailed("dotnet", $"restore \"{projectOrSolution}\"", exit); + } + + public void RestoreSolutions(IEnumerable solutions) + { + Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = 4 }, Restore); + } + + public void AnalyseSolutions(IEnumerable solutions) + { + Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = 4 } , solutionFile => + { + try + { + var sln = new SolutionFile(solutionFile); + progressMonitor.AnalysingSolution(solutionFile); + AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).Where(p => p.Exists).ToArray()); + } + catch (Microsoft.Build.Exceptions.InvalidProjectFileException ex) + { + progressMonitor.FailedProjectFile(solutionFile, ex.BaseMessage); + } + }); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs index 2c9e72c1eaa..02391c6eb7c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs @@ -5,17 +5,97 @@ using System.Xml; namespace Semmle.BuildAnalyser { + /// + /// A reference to a particular version of a particular package. + /// + class PackageReference + { + public PackageReference(string include, string version) { + Include = include; + Version = version; + } + public string Include, Version; + + public override string ToString() => $"Include={Include}, Version={Version}"; + } + + enum ProjectFileType + { + MsBuildProject, + DotNetProject, + OtherProject + } + + interface IProjectFile + { + IEnumerable ProjectReferences { get; } + + IEnumerable Packages { get; } + + IEnumerable References { get; } + + IEnumerable Sources { get; } + + IEnumerable TargetFrameworks { get; } + } + + class NetCoreProjectFile : IProjectFile + { + FileInfo path; + XmlDocument doc; + XmlElement root; + + public NetCoreProjectFile(FileInfo path) + { + this.path = path; + doc = new XmlDocument(); + doc.Load(path.FullName); + root = doc.DocumentElement; + } + + public IEnumerable ProjectReferences => throw new System.NotImplementedException(); + + public IEnumerable Packages + { + get + { + var packages = root.SelectNodes("/Project/ItemGroup/PackageReference"); + return packages.NodeList(). + Select(r => + new PackageReference(r.Attributes.GetNamedItem("Include").Value, r.Attributes.GetNamedItem("Version").Value)); + } + } + + public IEnumerable References => throw new System.NotImplementedException(); + + public IEnumerable Sources + { + get + { + return path.Directory.GetFiles("*.cs", SearchOption.AllDirectories); + } + } + + public IEnumerable TargetFrameworks => throw new System.NotImplementedException(); + } + /// /// Represents a .csproj file and reads information from it. /// class CsProjFile { + public string Filename { get; } + + public string Directory => Path.GetDirectoryName(Filename); + /// /// Reads the .csproj file. /// /// The .csproj file. public CsProjFile(FileInfo filename) { + Filename = filename.FullName; + try { // This can fail if the .csproj is invalid or has @@ -56,6 +136,8 @@ namespace Semmle.BuildAnalyser .ToArray(); } + string[] targetFrameworks = new string[0]; + /// /// Reads the .csproj file directly as XML. /// This doesn't handle variables etc, and should only used as a @@ -71,6 +153,35 @@ namespace Semmle.BuildAnalyser var projDir = filename.Directory; var root = projFile.DocumentElement; + // Figure out if it's dotnet core + + bool netCoreProjectFile = root.GetAttribute("Sdk") == "Microsoft.NET.Sdk"; + + if(netCoreProjectFile) + { + var frameworksNode = root.SelectNodes("/Project/PropertyGroup/TargetFrameworks").NodeList().Concat( + root.SelectNodes("/Project/PropertyGroup/TargetFramework").NodeList()).Select(node => node.InnerText); + + targetFrameworks = frameworksNode.SelectMany(node => node.Split(";")).ToArray(); + + var relativeCsIncludes2 = + root.SelectNodes("/Project/ItemGroup/Compile/@Include", mgr). + NodeList(). + Select(node => node.Value). + ToArray(); + + var explicitCsFiles = relativeCsIncludes2. + Select(cs => Path.DirectorySeparatorChar == '/' ? cs.Replace("\\", "/") : cs). + Select(f => Path.GetFullPath(Path.Combine(projDir.FullName, f))); + + var additionalCsFiles = System.IO.Directory.GetFiles(Directory, "*.cs", SearchOption.AllDirectories); + + csFiles = explicitCsFiles.Concat(additionalCsFiles).ToArray(); + + references = new string[0]; + return; + } + references = root.SelectNodes("/msbuild:Project/msbuild:ItemGroup/msbuild:Reference/@Include", mgr). NodeList(). @@ -97,6 +208,8 @@ namespace Semmle.BuildAnalyser /// public IEnumerable References => references; + public IEnumerable TargetFrameworks => targetFrameworks; + /// /// The list of C# source files in full path format. /// diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs new file mode 100644 index 00000000000..6d36892c7c9 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs @@ -0,0 +1,48 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; + +namespace Semmle.BuildAnalyser +{ + /// + /// Utilities to run the "dotnet" command. + /// + static class DotNet + { + public static int RestoreToDirectory(string projectOrSolutionFile, string packageDirectory) + { + using var proc = Process.Start("dotnet", $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\""); + proc.WaitForExit(); + return proc.ExitCode; + } + } + + /// + /// Utility to temporarily rename a set of files. + /// + class FileRenamer : IDisposable + { + string[] files; + const string suffix = ".codeqlhidden"; + + public FileRenamer(IEnumerable oldFiles) + { + files = oldFiles.Select(f => f.FullName).ToArray(); + + foreach(var file in files) + { + File.Move(file, file + suffix); + } + } + + public void Dispose() + { + foreach (var file in files) + { + File.Move(file + suffix, file); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNetRuntimeInfo.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNetRuntimeInfo.cs new file mode 100644 index 00000000000..21fe4a22338 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNetRuntimeInfo.cs @@ -0,0 +1,132 @@ +using System; +using System.Collections.Concurrent; +using System.Diagnostics; + +namespace RoslynWS +{ + /// + /// Information about the .NET Core runtime. + /// + public class DotNetRuntimeInfo + { + /// + /// A cache of .NET runtime information by target directory. + /// + static readonly ConcurrentDictionary _cache = new ConcurrentDictionary(); + + /// + /// The .NET Core version. + /// + public string Version { get; set; } + + /// + /// The .NET Core base directory. + /// + public string BaseDirectory { get; set; } + + /// + /// The current runtime identifier (RID). + /// + public string RID { get; set; } + + /// + /// Get information about the current .NET Core runtime. + /// + /// + /// An optional base directory where dotnet.exe should be run (this may affect the version it reports due to global.json). + /// + /// + /// A containing the runtime information. + /// + public static DotNetRuntimeInfo GetCurrent(string baseDirectory = null) + { + return _cache.GetOrAdd(baseDirectory, _ => + { + DotNetRuntimeInfo runtimeInfo = new DotNetRuntimeInfo(); + + Process dotnetInfoProcess = Process.Start(new ProcessStartInfo + { + FileName = "dotnet", + WorkingDirectory = baseDirectory, + Arguments = "--info", + UseShellExecute = false, + RedirectStandardOutput = true + }); + using (dotnetInfoProcess) + { + dotnetInfoProcess.WaitForExit(); + + string currentSection = null; + string currentLine; + while ((currentLine = dotnetInfoProcess.StandardOutput.ReadLine()) != null) + { + if (String.IsNullOrWhiteSpace(currentLine)) + continue; + + if (!currentLine.StartsWith(" ")) + { + currentSection = currentLine; + + continue; + } + + string[] property = currentLine.Split(new char[] { ':' }, count: 2); + if (property.Length != 2) + continue; + + property[0] = property[0].Trim(); + property[1] = property[1].Trim(); + + switch (currentSection) + { + case "Product Information:": + { + switch (property[0]) + { + case "Version": + { + runtimeInfo.Version = property[1]; + + break; + } + } + + break; + } + case "Runtime Environment:": + { + switch (property[0]) + { + case "Base Path": + { + runtimeInfo.BaseDirectory = property[1]; + + break; + } + case "RID": + { + runtimeInfo.RID = property[1]; + + break; + } + } + + break; + } + } + } + } + + return runtimeInfo; + }); + } + + /// + /// Clear the cache of .NET runtime information. + /// + public static void ClearCache() + { + _cache.Clear(); + } + } +} \ No newline at end of file diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/MSBuildHelper.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/MSBuildHelper.cs new file mode 100644 index 00000000000..cbba36e8419 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/MSBuildHelper.cs @@ -0,0 +1,260 @@ + +using Microsoft.Build.Construction; +using Microsoft.Build.Evaluation; +using Microsoft.Build.Exceptions; +using System; +using System.Collections.Generic; +using System.IO; +using System.Collections.Immutable; + +namespace RoslynWS +{ + /// + /// Helper methods for working with MSBuild projects. + /// + public static class MSBuildHelper + { + /// + /// The names of well-known item metadata. + /// + public static readonly ImmutableSortedSet WellknownMetadataNames = + ImmutableSortedSet.Create( + "FullPath", + "RootDir", + "Filename", + "Extension", + "RelativeDir", + "Directory", + "RecursiveDir", + "Identity", + "ModifiedTime", + "CreatedTime", + "AccessedTime" + ); + + /// + /// Create an MSBuild project collection. + /// + /// + /// The base (i.e. solution) directory. + /// + /// + /// The project collection. + /// + public static ProjectCollection CreateProjectCollection(string solutionDirectory) + { + return CreateProjectCollection(solutionDirectory, + DotNetRuntimeInfo.GetCurrent(solutionDirectory) + ); + } + + /// + /// Create an MSBuild project collection. + /// + /// + /// The base (i.e. solution) directory. + /// + /// + /// Information about the current .NET Core runtime. + /// + /// + /// The project collection. + /// + public static ProjectCollection CreateProjectCollection(string solutionDirectory, DotNetRuntimeInfo runtimeInfo) + { + if (String.IsNullOrWhiteSpace(solutionDirectory)) + throw new ArgumentException("Argument cannot be null, empty, or entirely composed of whitespace: 'baseDir'.", nameof(solutionDirectory)); + + if (runtimeInfo == null) + throw new ArgumentNullException(nameof(runtimeInfo)); + + if (String.IsNullOrWhiteSpace(runtimeInfo.BaseDirectory)) + throw new InvalidOperationException("Cannot determine base directory for .NET Core."); + + Dictionary globalProperties = CreateGlobalMSBuildProperties(runtimeInfo, solutionDirectory); + EnsureMSBuildEnvironment(globalProperties); + + ProjectCollection projectCollection = new ProjectCollection(globalProperties) { IsBuildEnabled = false }; + + // Override toolset paths (for some reason these point to the main directory where the dotnet executable lives). + Toolset toolset = projectCollection.GetToolset("15.0"); + toolset = new Toolset( + toolsVersion: "15.0", + toolsPath: globalProperties["MSBuildExtensionsPath"], + projectCollection: projectCollection, + msbuildOverrideTasksPath: "" + ); + projectCollection.AddToolset(toolset); + + return projectCollection; + } + + /// + /// Create global properties for MSBuild. + /// + /// + /// Information about the current .NET Core runtime. + /// + /// + /// The base (i.e. solution) directory. + /// + /// + /// A dictionary containing the global properties. + /// + public static Dictionary CreateGlobalMSBuildProperties(DotNetRuntimeInfo runtimeInfo, string solutionDirectory) + { + if (runtimeInfo == null) + throw new ArgumentNullException(nameof(runtimeInfo)); + + if (String.IsNullOrWhiteSpace(solutionDirectory)) + throw new ArgumentException("Argument cannot be null, empty, or entirely composed of whitespace: 'solutionDirectory'.", nameof(solutionDirectory)); + + if (solutionDirectory.Length > 0 && solutionDirectory[solutionDirectory.Length - 1] != Path.DirectorySeparatorChar) + solutionDirectory += Path.DirectorySeparatorChar; + + return new Dictionary + { + [WellKnownPropertyNames.DesignTimeBuild] = "true", + [WellKnownPropertyNames.BuildProjectReferences] = "false", + [WellKnownPropertyNames.ResolveReferenceDependencies] = "true", + [WellKnownPropertyNames.SolutionDir] = solutionDirectory, + [WellKnownPropertyNames.MSBuildExtensionsPath] = runtimeInfo.BaseDirectory, + [WellKnownPropertyNames.MSBuildSDKsPath] = Path.Combine(runtimeInfo.BaseDirectory, "Sdks"), + [WellKnownPropertyNames.RoslynTargetsPath] = Path.Combine(runtimeInfo.BaseDirectory, "Roslyn") + }; + } + + /// + /// Ensure that environment variables are populated using the specified MSBuild global properties. + /// + /// + /// The MSBuild global properties + /// + public static void EnsureMSBuildEnvironment(Dictionary globalMSBuildProperties) + { + if (globalMSBuildProperties == null) + throw new ArgumentNullException(nameof(globalMSBuildProperties)); + + // Kinda sucks that the simplest way to get MSBuild to resolve SDKs correctly is using environment variables, but there you go. + Environment.SetEnvironmentVariable( + WellKnownPropertyNames.MSBuildExtensionsPath, + globalMSBuildProperties[WellKnownPropertyNames.MSBuildExtensionsPath] + ); + Environment.SetEnvironmentVariable( + WellKnownPropertyNames.MSBuildSDKsPath, + globalMSBuildProperties[WellKnownPropertyNames.MSBuildSDKsPath] + ); + } + + /// + /// Does the specified property name represent a private property? + /// + /// + /// The property name. + /// + /// + /// true, if the property name starts with an underscore; otherwise, false. + /// + public static bool IsPrivateProperty(string propertyName) => propertyName?.StartsWith("_") ?? false; + + /// + /// Does the specified metadata name represent a private property? + /// + /// + /// The metadata name. + /// + /// + /// true, if the metadata name starts with an underscore; otherwise, false. + /// + public static bool IsPrivateMetadata(string metadataName) => metadataName?.StartsWith("_") ?? false; + + /// + /// Does the specified item type represent a private property? + /// + /// + /// The item type. + /// + /// + /// true, if the item type starts with an underscore; otherwise, false. + /// + public static bool IsPrivateItemType(string itemType) => itemType?.StartsWith("_") ?? false; + + /// + /// Determine whether the specified metadata name represents well-known (built-in) item metadata. + /// + /// + /// The metadata name. + /// + /// + /// true, if represents well-known item metadata; otherwise, false. + /// + public static bool IsWellKnownItemMetadata(string metadataName) => WellknownMetadataNames.Contains(metadataName); + + /// + /// Create a copy of the project for caching. + /// + /// + /// The MSBuild project. + /// + /// + /// The project copy (independent of original, but sharing the same ). + /// + /// + /// You can only create a single cached copy for a given project. + /// + public static Project CloneAsCachedProject(this Project project) + { + if (project == null) + throw new ArgumentNullException(nameof(project)); + + ProjectRootElement clonedXml = project.Xml.DeepClone(); + Project clonedProject = new Project(clonedXml, project.GlobalProperties, project.ToolsVersion, project.ProjectCollection); + clonedProject.FullPath = Path.ChangeExtension(project.FullPath, + ".cached" + Path.GetExtension(project.FullPath) + ); + + return clonedProject; + } + + /// + /// The names of well-known MSBuild properties. + /// + public static class WellKnownPropertyNames + { + /// + /// The "MSBuildExtensionsPath" property. + /// + public static readonly string MSBuildExtensionsPath = "MSBuildExtensionsPath"; + + /// + /// The "MSBuildSDKsPath" property. + /// + public static readonly string MSBuildSDKsPath = "MSBuildSDKsPath"; + + /// + /// The "SolutionDir" property. + /// + public static readonly string SolutionDir = "SolutionDir"; + + /// + /// The "_ResolveReferenceDependencies" property. + /// + public static readonly string ResolveReferenceDependencies = "_ResolveReferenceDependencies"; + + /// + /// The "DesignTimeBuild" property. + /// + public static readonly string DesignTimeBuild = "DesignTimeBuild"; + + /// + /// The "BuildProjectReferences" property. + /// + public static readonly string BuildProjectReferences = "BuildProjectReferences"; + + /// + /// The "RoslynTargetsPath" property. + /// + public static readonly string RoslynTargetsPath = "RoslynTargetsPath"; + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackageRepository.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackageRepository.cs new file mode 100644 index 00000000000..3b84df7463c --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackageRepository.cs @@ -0,0 +1,357 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; + +namespace Semmle.BuildAnalyser +{ + /// + /// A package in a NuGet package repository. + /// For example, the directory C:\Users\calum\.nuget\packages\microsoft.visualbasic. + /// + /// Each package contains a number of subdirectories, organised by version. + /// + class Package + { + public readonly string directory; + + /// + /// Constructs a package for the given directory. + /// + /// The directory. + public Package(string dir) + { + directory = dir; + } + + /// + /// The name of the package. + /// + public string Name => Path.GetDirectoryName(directory); + + /// + /// The versions that exist within the package. + /// + public IEnumerable Versions => Directory.GetDirectories(directory).Select(dir => new PackageVersion(dir)); + + public override string ToString() => Path.GetFileName(directory); + + public PackageVersion FindVersion(string version) + { + if (Directory.Exists(Path.Combine(directory, version))) + return new PackageVersion(Path.Combine(directory, version)); + else + return Versions.OrderByDescending(v => v.Version).First(); + } + + /// + /// Locates the exact version of a particular package. + /// + /// The version to locate. + /// The specific version of the package. + public PackageVersion FindExactVersion(string version) + { + if (Directory.Exists(Path.Combine(directory, version))) + return new PackageVersion(Path.Combine(directory, version)); + else + return null; + } + } + + /// + /// A package in a NuGet package respository, including the specific version. + /// For example, the directory C:\Users\calum\.nuget\packages\microsoft.visualbasic\10.0.1 + /// + class PackageVersion + { + readonly string directory; + + /// + /// The version of the package. + /// + public string Version => Path.GetFileName(directory); + + /// + /// Constructs a package version from its directory. + /// + /// The directory of this package. + public PackageVersion(string directory) + { + if (!Directory.Exists(directory)) + throw new DirectoryNotFoundException(directory); + + this.directory = directory; + } + + public override string ToString() => Version; + + /// + /// The frameworks within this package. + /// Sometimes a directory references several frameworks, for example + /// "net451+netstandard2.0". This are split into separate frameworks. + /// + IEnumerable UnorderedFrameworks + { + get + { + return UnorderedFrameworksInDirectory(Path.Combine(directory, "lib")). + Concat(UnorderedFrameworksInDirectory(Path.Combine(directory, "ref"))). + Concat(UnorderedFrameworksInDirectory(Path.Combine(directory, "build", "lib"))). + Concat(UnorderedFrameworksInDirectory(Path.Combine(directory, "build", "ref"))). + Concat(TryDirectory(Path.Combine(directory, "lib"))); + } + } + + IEnumerable TryDirectory(string directory) + { + if (Directory.Exists(directory)) + yield return new PackageFramework(directory, "unknown"); + } + + IEnumerable UnorderedFrameworksInDirectory(string lib) + { + if (!Directory.Exists(lib)) + yield break; + foreach (var p in System.IO.Directory.GetDirectories(lib)) + { + var name = Path.GetFileName(p); + if (name.Contains('+')) + { + foreach (var p2 in name.Split('+')) + yield return new PackageFramework(p, p2); + } + else + yield return new PackageFramework(p, name); + } + } + + /// + /// The frameworks in this package, in a consistent sequence, with the "best" frameworks + /// appearing at the start of the list. + /// + /// Priorities "netstandard" framework, followed by "netcoreapp", followed by everything else. + /// Then, selects the highest version number. + /// + public IEnumerable Frameworks => + UnorderedFrameworks. + OrderBy(framework => framework.Framework.StartsWith("netstandard") ? 0 : framework.Framework.StartsWith("netcoreapp") ? 1 : 2). + ThenByDescending(framework => framework.Framework); + + /// + /// Finds the best framework containing references. + /// Returns null if no suitable framework was found. + /// + public PackageFramework? BestFramework => Frameworks.Where(f => f.References.Any()).FirstOrDefault(); + + public bool ContainsLibraries + { + get + { + return Directory.Exists(Path.Combine(directory, "lib")) || + Directory.Exists(Path.Combine(directory, "ref")) || + Directory.Exists(Path.Combine(directory, "build", "lib")) || + Directory.Exists(Path.Combine(directory, "build", "ref")); + } + } + + public bool ContainsDLLs + { + get + { + return Directory.GetFiles(directory, "*.dll", SearchOption.AllDirectories).Any(); + } + } + } + + /// + /// A framework in a package. + /// For example, C:\Users\calum\.nuget\packages\microsoft.testplatform.objectmodel\16.4.0\lib\netstandard2.0 + /// + class PackageFramework + { + public string Directory { get; } + + /// + /// The framework name. + /// + public string Framework { get; } + + /// + /// Constructs a package framework from a directory. + /// The framework is needed because the directory may specify more than one framework. + /// + /// The directory path. + /// The framework. + public PackageFramework(string dir, string framework) + { + if (!System.IO.Directory.Exists(dir)) + throw new FileNotFoundException(dir); + Directory = dir; + Framework = framework; + } + + /// + /// The reference DLLs contained within the directory. + /// + public IEnumerable References + { + get + { + return new DirectoryInfo(Directory).GetFiles("*.dll").Select(fi => fi.FullName); + } + } + + public override string ToString() => Directory; + } + + class TemporaryDirectory : IDisposable + { + readonly IProgressMonitor ProgressMonitor; + + public DirectoryInfo DirInfo { get; } + + public TemporaryDirectory(string name, IProgressMonitor pm) + { + ProgressMonitor = pm; + DirInfo = new DirectoryInfo(name); + DirInfo.Create(); + } + + /// + /// Computes a unique temp directory for the packages associated + /// with this source tree. Use a SHA1 of the directory name. + /// + /// + /// The full path of the temp directory. + public static string ComputeTempDirectory(string srcDir) + { + var bytes = Encoding.Unicode.GetBytes(srcDir); + + var sha1 = new SHA1CryptoServiceProvider(); + var sha = sha1.ComputeHash(bytes); + var sb = new StringBuilder(); + foreach (var b in sha.Take(8)) + sb.AppendFormat("{0:x2}", b); + + return Path.Combine(Path.GetTempPath(), "GitHub", "packages", sb.ToString()); + } + + public static TemporaryDirectory CreateTempDirectory(string source, IProgressMonitor pm) => new TemporaryDirectory(ComputeTempDirectory(source), pm); + + public void Cleanup() + { + try + { + DirInfo.Delete(true); + } + catch (System.IO.IOException ex) + { + ProgressMonitor.Warning(string.Format("Couldn't delete package directory - it's probably held open by something else: {0}", ex.Message)); + } + + } + + public void Dispose() + { + Cleanup(); + } + + public override string ToString() => DirInfo.FullName.ToString(); + } + + /// + /// The NuGet package repository. + /// + class NugetPackageRepository + { + // A list of package directories, in the order they should be searched. + private readonly string[] packageDirs; + + public NugetPackageRepository(params string[] dirs) + { + packageDirs = dirs; + } + + /// + /// Constructs a NuGet package repository, using the default locations. + /// For example, + /// $HOME/.nuget/packages, /usr/share/dotnet/sdk/NuGetFallbackFolder + /// + public NugetPackageRepository(string sourceDir) + { + var homeFolder = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + var nugetPackages = Path.Combine(homeFolder, ".nuget", "packages"); + var nugetFallbackFolder = Environment.OSVersion.Platform == PlatformID.Win32NT ? + @"C:\Program Files\dotnet\sdk\NuGetFallbackFolder" : + "/usr/share/dotnet/sdk/NuGetFallbackFolder"; + + packageDirs = new string[] { nugetPackages, nugetFallbackFolder }; + } + + /// + /// Enumerate all available packages. + /// + public IEnumerable Packages + { + get + { + foreach (var d in packageDirs) + foreach (var p in Directory.GetDirectories(d)) + { + var name = Path.GetFileName(p); + if (!name.StartsWith('.')) + yield return new Package(p); + } + } + } + + /// + /// Tries to find a PackageFramework directory for a given package reference. + /// + /// The package reference to search for. + /// The package that was found. + /// True if a package/version/framework was found. + public bool TryFindLibs(PackageReference reference, out PackageFramework package, out ResolutionFailureReason reason) + { + var packages = packageDirs. + Where(d => Directory.Exists(Path.Combine(d, reference.Include.ToLowerInvariant()))). + Select(d => new Package(Path.Combine(d, reference.Include.ToLowerInvariant()))); + + if(!packages.Any()) + { + reason = ResolutionFailureReason.PackageNotFound; + package = null; + return false; + } + + var version = packages.Select(p => p.FindVersion(reference.Version)).FirstOrDefault(v => !(v is null)); + + if (version is null) + { + reason = ResolutionFailureReason.VersionNotFound; + package = null; + return false; + } + + package = version.BestFramework; + if(package is null) + { + reason = ResolutionFailureReason.LibsNotFound; + return false; + } + + reason = ResolutionFailureReason.Success; + return true; + } + } + + enum ResolutionFailureReason + { + Success, + PackageNotFound, + VersionNotFound, + LibsNotFound + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs index 1f0755f307f..0fc207f2049 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs @@ -19,10 +19,10 @@ namespace Semmle.BuildAnalyser /// Create the package manager for a specified source tree. /// /// The source directory. - public NugetPackages(string sourceDir) + public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory) { SourceDirectory = sourceDir; - PackageDirectory = computeTempDirectory(sourceDir); + PackageDirectory = packageDirectory; // Expect nuget.exe to be in a `nuget` directory under the directory containing this exe. var currentAssembly = System.Reflection.Assembly.GetExecutingAssembly().Location; @@ -50,45 +50,12 @@ namespace Semmle.BuildAnalyser /// public IEnumerable PackageFiles => packages; - // Whether to delete the packages directory prior to each run. - // Makes each build more reproducible. - const bool cleanupPackages = true; - - public void Cleanup(IProgressMonitor pm) - { - var packagesDirectory = new DirectoryInfo(PackageDirectory); - - if (packagesDirectory.Exists) - { - try - { - packagesDirectory.Delete(true); - } - catch (System.IO.IOException ex) - { - pm.Warning(string.Format("Couldn't delete package directory - it's probably held open by something else: {0}", ex.Message)); - } - } - } - /// /// Download the packages to the temp folder. /// /// The progress monitor used for reporting errors etc. public void InstallPackages(IProgressMonitor pm) { - if (cleanupPackages) - { - Cleanup(pm); - } - - var packagesDirectory = new DirectoryInfo(PackageDirectory); - - if (!Directory.Exists(PackageDirectory)) - { - packagesDirectory.Create(); - } - foreach (var package in packages) { RestoreNugetPackage(package.FullName, pm); @@ -109,31 +76,7 @@ namespace Semmle.BuildAnalyser /// This will be in the Temp location /// so as to not trample the source tree. /// - public string PackageDirectory - { - get; - private set; - } - - readonly SHA1CryptoServiceProvider sha1 = new SHA1CryptoServiceProvider(); - - /// - /// Computes a unique temp directory for the packages associated - /// with this source tree. Use a SHA1 of the directory name. - /// - /// - /// The full path of the temp directory. - string computeTempDirectory(string srcDir) - { - var bytes = Encoding.Unicode.GetBytes(srcDir); - - var sha = sha1.ComputeHash(bytes); - var sb = new StringBuilder(); - foreach (var b in sha.Take(8)) - sb.AppendFormat("{0:x2}", b); - - return Path.Combine(Path.GetTempPath(), "Semmle", "packages", sb.ToString()); - } + public TemporaryDirectory PackageDirectory { get; } /// /// Restore all files in a specified package. diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs index e0367fa63c1..3ba368f6da0 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs @@ -3,6 +3,11 @@ using System.Collections.Generic; using System.Linq; using Semmle.BuildAnalyser; using Semmle.Util.Logging; +using System.IO; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Text; + +// using Microsoft.Build.Locator; namespace Semmle.Extraction.CSharp.Standalone { @@ -82,9 +87,15 @@ namespace Semmle.Extraction.CSharp.Standalone public class Program { + void LoadSolutionFile(string file) + { + + } + static int Main(string[] args) { var options = Options.Create(args); + options.CIL = true; var output = new ConsoleLogger(options.Verbosity); var a = new Analysis(output); @@ -97,6 +108,8 @@ namespace Semmle.Extraction.CSharp.Standalone if (options.Errors) return 1; + var start = DateTime.Now; + output.Log(Severity.Info, "Running C# standalone extractor"); a.AnalyseProjects(options); int sourceFiles = a.Extraction.Sources.Count(); @@ -117,7 +130,7 @@ namespace Semmle.Extraction.CSharp.Standalone new ExtractionProgress(output), new FileLogger(options.Verbosity, Extractor.GetCSharpLogPath()), options); - output.Log(Severity.Info, "Extraction complete"); + output.Log(Severity.Info, $"Extraction completed in {DateTime.Now-start}"); } a.Cleanup(); @@ -151,7 +164,7 @@ namespace Semmle.Extraction.CSharp.Standalone public void MissingSummary(int missingTypes, int missingNamespaces) { - logger.Log(Severity.Info, "Failed to resolve {0} types and {1} namespaces", missingTypes, missingNamespaces); + logger.Log(Severity.Info, "Failed to resolve {0} types in {1} namespaces", missingTypes, missingNamespaces); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs index f4bde55ec55..a004e59eb13 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs @@ -10,6 +10,7 @@ namespace Semmle.BuildAnalyser void FindingFiles(string dir); void UnresolvedReference(string id, string project); void AnalysingProjectFiles(int count); + void AnalysingSolution(string filename); void FailedProjectFile(string filename, string reason); void FailedNugetCommand(string exe, string args, string message); void NugetInstall(string package); @@ -18,6 +19,11 @@ namespace Semmle.BuildAnalyser void Warning(string message); void ResolvedConflict(string asm1, string asm2); void MissingProject(string projectFile); + void Restored(string line); + void MissingPackage(string package, string version, ResolutionFailureReason reason); + void FoundPackage(string package, string version, string directory); + void CommandFailed(string exe, string arguments, int exitCode); + void MissingNuGet(); } class ProgressMonitor : IProgressMonitor @@ -51,6 +57,11 @@ namespace Semmle.BuildAnalyser logger.Log(Severity.Info, "Analyzing project files..."); } + public void AnalysingSolution(string filename) + { + logger.Log(Severity.Info, $"Analysing {filename}..."); + } + public void FailedProjectFile(string filename, string reason) { logger.Log(Severity.Info, "Couldn't read project file {0}: {1}", filename, reason); @@ -73,7 +84,8 @@ namespace Semmle.BuildAnalyser } public void Summary(int existingSources, int usedSources, int missingSources, - int references, int unresolvedReferences, int resolvedConflicts, int totalProjects, int failedProjects) + int references, int unresolvedReferences, + int resolvedConflicts, int totalProjects, int failedProjects) { logger.Log(Severity.Info, ""); logger.Log(Severity.Info, "Build analysis summary:"); @@ -87,6 +99,23 @@ namespace Semmle.BuildAnalyser logger.Log(Severity.Info, "{0, 6} missing/failed projects", failedProjects); } + public void Restored(string line) + { + logger.Log(Severity.Debug, $" {line}"); + } + + private static string[] reasonText = { "success", "package was not found", "the version was not found", "the package does not appear to contain any libraries" }; + + public void MissingPackage(string package, string version, ResolutionFailureReason reason) + { + logger.Log(Severity.Info, $" Couldn't find package {package} {version} because {reasonText[(int)reason]}"); + } + + public void FoundPackage(string package, string version, string directory) + { + logger.Log(Severity.Debug, $" Found package {package} {version} in {directory}"); + } + public void Warning(string message) { logger.Log(Severity.Warning, message); @@ -94,12 +123,22 @@ namespace Semmle.BuildAnalyser public void ResolvedConflict(string asm1, string asm2) { - logger.Log(Severity.Info, "Resolved {0} as {1}", asm1, asm2); + logger.Log(Severity.Debug, "Resolved {0} as {1}", asm1, asm2); } public void MissingProject(string projectFile) { logger.Log(Severity.Info, "Solution is missing {0}", projectFile); } + + public void CommandFailed(string exe, string arguments, int exitCode) + { + logger.Log(Severity.Error, $"Command {exe} {arguments} failed with exit code {exitCode}"); + } + + public void MissingNuGet() + { + logger.Log(Severity.Error, "Missing nuget.exe"); + } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj index 4cf0274b737..1d4ab57d2d8 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj @@ -1,4 +1,4 @@ - + Exe @@ -21,7 +21,10 @@ - + + + + diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/SolutionFile.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/SolutionFile.cs index b1a3edd4cf6..ffaebe360fe 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/SolutionFile.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/SolutionFile.cs @@ -11,6 +11,7 @@ namespace Semmle.BuildAnalyser class SolutionFile { readonly Microsoft.Build.Construction.SolutionFile solutionFile; + public string FullPath { get; } /// /// Read the file. @@ -19,8 +20,8 @@ namespace Semmle.BuildAnalyser public SolutionFile(string filename) { // SolutionFile.Parse() expects a rooted path. - var fullPath = Path.GetFullPath(filename); - solutionFile = Microsoft.Build.Construction.SolutionFile.Parse(fullPath); + FullPath = Path.GetFullPath(filename); + solutionFile = Microsoft.Build.Construction.SolutionFile.Parse(FullPath); } /// diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Access.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Access.cs index 0488fe84ffe..6962e8381d9 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Access.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Access.cs @@ -45,7 +45,10 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions Access(ExpressionNodeInfo info, ISymbol symbol, bool implicitThis, IEntity target) : base(info.SetKind(AccessKind(info.Context, symbol))) { - cx.TrapWriter.Writer.expr_access(this, target); + if (!(target is null)) + { + cx.TrapWriter.Writer.expr_access(this, target); + } if (implicitThis && !symbol.IsStatic) { diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MemberAccess.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MemberAccess.cs index e41ef0edf23..0bc84ca9c0c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MemberAccess.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MemberAccess.cs @@ -71,7 +71,9 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions if (symbol == null) { info.Context.ModelError(info.Node, "Failed to determine symbol for member access"); - return new MemberAccess(info.SetKind(ExprKind.UNKNOWN), expression, symbol); + // Default to property access - this can still give useful results but + // the target of the expression should be checked in QL. + return new MemberAccess(info.SetKind(ExprKind.PROPERTY_ACCESS), expression, symbol); } ExprKind kind; diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs index e22d32c0d01..cecec5bc028 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs @@ -25,7 +25,7 @@ namespace Semmle.Extraction.CSharp.Entities { if (symbol.TypeKind == TypeKind.Error) { - Context.Extractor.MissingType(symbol.ToString()); + Context.Extractor.MissingType(symbol.ToString(), Context.FromSource); return; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs index 4fdbe7c18ad..02b67efc164 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs @@ -32,7 +32,7 @@ namespace Semmle.Extraction.CSharp.Entities if (namespaceSymbol == null) { - cx.Extractor.MissingNamespace(Node.Name.ToFullString()); + cx.Extractor.MissingNamespace(Node.Name.ToFullString(), cx.FromSource); cx.ModelError(Node, "Namespace not found"); return; } diff --git a/csharp/extractor/Semmle.Extraction/Context.cs b/csharp/extractor/Semmle.Extraction/Context.cs index 93f99381858..918642f198d 100644 --- a/csharp/extractor/Semmle.Extraction/Context.cs +++ b/csharp/extractor/Semmle.Extraction/Context.cs @@ -155,7 +155,7 @@ namespace Semmle.Extraction #if DEBUG_LABELS using (var id = new StringWriter()) { - entity.WriteId(id); + entity.WriteQuotedId(id); CheckEntityHasUniqueLabel(id.ToString(), entity); } #endif @@ -270,6 +270,8 @@ namespace Semmle.Extraction TrapWriter = trapWriter; } + public bool FromSource => Scope.FromSource; + public bool IsGlobalContext => Scope.IsGlobalScope; public readonly ICommentGenerator CommentGenerator = new CommentProcessor(); diff --git a/csharp/extractor/Semmle.Extraction/ExtractionScope.cs b/csharp/extractor/Semmle.Extraction/ExtractionScope.cs index 7f4f599fe5c..60daff8d013 100644 --- a/csharp/extractor/Semmle.Extraction/ExtractionScope.cs +++ b/csharp/extractor/Semmle.Extraction/ExtractionScope.cs @@ -25,6 +25,8 @@ namespace Semmle.Extraction bool InFileScope(string path); bool IsGlobalScope { get; } + + bool FromSource { get; } } /// @@ -49,6 +51,8 @@ namespace Semmle.Extraction public bool InScope(ISymbol symbol) => SymbolEqualityComparer.Default.Equals(symbol.ContainingAssembly, assembly) || SymbolEqualityComparer.Default.Equals(symbol, assembly); + + public bool FromSource => false; } /// @@ -68,5 +72,7 @@ namespace Semmle.Extraction public bool InFileScope(string path) => path == sourceTree.FilePath; public bool InScope(ISymbol symbol) => symbol.Locations.Any(loc => loc.SourceTree == sourceTree); + + public bool FromSource => true; } } diff --git a/csharp/extractor/Semmle.Extraction/Extractor.cs b/csharp/extractor/Semmle.Extraction/Extractor.cs index e470d3258ec..e1ca23f645b 100644 --- a/csharp/extractor/Semmle.Extraction/Extractor.cs +++ b/csharp/extractor/Semmle.Extraction/Extractor.cs @@ -50,13 +50,15 @@ namespace Semmle.Extraction /// Record a new error type. /// /// The display name of the type, qualified where possible. - void MissingType(string fqn); + /// The missing type was referenced from a source file. + void MissingType(string fqn, bool fromSource); /// /// Record an unresolved `using namespace` directive. /// /// The full name of the namespace. - void MissingNamespace(string fqn); + /// The missing namespace was referenced from a source file. + void MissingNamespace(string fqn, bool fromSource); /// /// The list of missing types. @@ -167,16 +169,22 @@ namespace Semmle.Extraction readonly ISet missingTypes = new SortedSet(); readonly ISet missingNamespaces = new SortedSet(); - public void MissingType(string fqn) + public void MissingType(string fqn, bool fromSource) { - lock (mutex) - missingTypes.Add(fqn); + if (fromSource) + { + lock (mutex) + missingTypes.Add(fqn); + } } - public void MissingNamespace(string fqdn) + public void MissingNamespace(string fqdn, bool fromSource) { - lock (mutex) - missingNamespaces.Add(fqdn); + if (fromSource) + { + lock (mutex) + missingNamespaces.Add(fqdn); + } } public Context CreateContext(Compilation c, TrapWriter trapWriter, IExtractionScope scope)