From 87970337aefb96ec910714c0196ac3bed7bab1e8 Mon Sep 17 00:00:00 2001 From: Calum Grant Date: Tue, 7 Jan 2020 18:21:23 +0000 Subject: [PATCH 1/5] C#: Improvements to buildless extraction, particularly for .NET Core. --- .gitignore | 3 +- .../AssemblyCache.cs | 14 +- .../BuildAnalysis.cs | 182 +++++---- .../CsProjFile.cs | 113 ++++++ .../DotNet.cs | 48 +++ .../DotNetRuntimeInfo.cs | 132 +++++++ .../MSBuildHelper.cs | 260 +++++++++++++ .../NugetPackageRepository.cs | 357 ++++++++++++++++++ .../NugetPackages.cs | 63 +--- .../Program.cs | 17 +- .../ProgressMonitor.cs | 43 ++- ...Semmle.Extraction.CSharp.Standalone.csproj | 7 +- .../SolutionFile.cs | 5 +- .../Entities/Expressions/Access.cs | 5 +- .../Entities/Expressions/MemberAccess.cs | 4 +- .../Entities/Types/NamedType.cs | 2 +- .../Entities/UsingDirective.cs | 2 +- csharp/extractor/Semmle.Extraction/Context.cs | 4 +- .../Semmle.Extraction/ExtractionScope.cs | 6 + .../extractor/Semmle.Extraction/Extractor.cs | 24 +- 20 files changed, 1140 insertions(+), 151 deletions(-) create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNetRuntimeInfo.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.Standalone/MSBuildHelper.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackageRepository.cs diff --git a/.gitignore b/.gitignore index 1768ebb161f..c0e9ed803ff 100644 --- a/.gitignore +++ b/.gitignore @@ -16,5 +16,6 @@ # It's useful (though not required) to be able to unpack codeql in the ql checkout itself /codeql/ -.vscode/settings.json + csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/launchSettings.json +.vscode diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs index db2664bf4c9..f93911b8a38 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs @@ -163,7 +163,19 @@ namespace Semmle.BuildAnalyser /// /// The filename to query. /// The assembly info. - public AssemblyInfo GetAssemblyInfo(string filepath) => assemblyInfo[filepath]; + public AssemblyInfo GetAssemblyInfo(string filepath) + { + if(assemblyInfo.TryGetValue(filepath, out var info)) + { + return info; + } + else + { + info = AssemblyInfo.ReadFromFile(filepath); + assemblyInfo.Add(filepath, info); + return info; + } + } // List of pending DLLs to index. readonly List dlls = new List(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs index b0ef328bcd0..dfc110c0d03 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs @@ -5,6 +5,8 @@ using System.Linq; using System.Runtime.InteropServices; using Semmle.Util; using Semmle.Extraction.CSharp.Standalone; +using System.Threading.Tasks; +using System.Collections.Concurrent; namespace Semmle.BuildAnalyser { @@ -56,6 +58,7 @@ namespace Semmle.BuildAnalyser int failedProjects, succeededProjects; readonly string[] allSources; int conflictedReferences = 0; + object mutex = new object(); /// /// Performs a C# build analysis. @@ -64,6 +67,8 @@ namespace Semmle.BuildAnalyser /// Display of analysis progress. public BuildAnalysis(Options options, IProgressMonitor progress) { + var startTime = DateTime.Now; + progressMonitor = progress; sourceDir = new DirectoryInfo(options.SrcDir); @@ -74,31 +79,43 @@ namespace Semmle.BuildAnalyser Where(d => !options.ExcludesFile(d)). ToArray(); - var dllDirNames = options.DllDirs.Select(Path.GetFullPath); + var dllDirNames = options.DllDirs.Select(Path.GetFullPath).ToList(); + PackageDirectory = TemporaryDirectory.CreateTempDirectory(sourceDir.FullName, progressMonitor); if (options.UseNuGet) { - nuget = new NugetPackages(sourceDir.FullName); - ReadNugetFiles(); - dllDirNames = dllDirNames.Concat(Enumerators.Singleton(nuget.PackageDirectory)); + try + { + nuget = new NugetPackages(sourceDir.FullName, PackageDirectory); + ReadNugetFiles(); + } + catch(FileNotFoundException) + { + progressMonitor.MissingNuGet(); + } } // Find DLLs in the .Net Framework if (options.ScanNetFrameworkDlls) { - dllDirNames = dllDirNames.Concat(Runtime.Runtimes.Take(1)); + dllDirNames.Add(Runtime.Runtimes.First()); } - - assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress); - - // Analyse all .csproj files in the source tree. - if (options.SolutionFile != null) + { - AnalyseSolution(options.SolutionFile); - } - else if (options.AnalyseCsProjFiles) - { - AnalyseProjectFiles(); + using var renamer1 = new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories)); + using var renamer2 = new FileRenamer(sourceDir.GetFiles("Directory.Build.props", SearchOption.AllDirectories)); + + var solutions = options.SolutionFile != null ? + new[] { options.SolutionFile } : + sourceDir.GetFiles("*.sln", SearchOption.AllDirectories).Select(d => d.FullName); + + + RestoreSolutions(solutions); + dllDirNames.Add(PackageDirectory.DirInfo.FullName); + assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress); + AnalyseSolutions(solutions); + + usedReferences = new HashSet(assemblyCache.AllAssemblies.Select(a => a.Filename)); } if (!options.AnalyseCsProjFiles) @@ -106,6 +123,7 @@ namespace Semmle.BuildAnalyser usedReferences = new HashSet(assemblyCache.AllAssemblies.Select(a => a.Filename)); } + ResolveConflicts(); if (options.UseMscorlib) @@ -133,6 +151,8 @@ namespace Semmle.BuildAnalyser conflictedReferences, succeededProjects + failedProjects, failedProjects); + + Console.WriteLine($"Build analysis completed in {DateTime.Now - startTime}"); } /// @@ -183,7 +203,8 @@ namespace Semmle.BuildAnalyser /// The filename of the reference. void UseReference(string reference) { - usedReferences.Add(reference); + lock (mutex) + usedReferences.Add(reference); } /// @@ -194,11 +215,13 @@ namespace Semmle.BuildAnalyser { if (sourceFile.Exists) { - usedSources.Add(sourceFile.FullName); + lock(mutex) + usedSources.Add(sourceFile.FullName); } else { - missingSources.Add(sourceFile.FullName); + lock(mutex) + missingSources.Add(sourceFile.FullName); } } @@ -236,59 +259,63 @@ namespace Semmle.BuildAnalyser /// The project file making the reference. void UnresolvedReference(string id, string projectFile) { - unresolvedReferences[id] = projectFile; + lock(mutex) + unresolvedReferences[id] = projectFile; } - /// - /// Performs an analysis of all .csproj files. - /// - void AnalyseProjectFiles() - { - AnalyseProjectFiles(sourceDir.GetFiles("*.csproj", SearchOption.AllDirectories)); - } + TemporaryDirectory PackageDirectory; /// /// Reads all the source files and references from the given list of projects. /// /// The list of projects to analyse. - void AnalyseProjectFiles(FileInfo[] projectFiles) + void AnalyseProjectFiles(IEnumerable projectFiles) { - progressMonitor.AnalysingProjectFiles(projectFiles.Count()); - foreach (var proj in projectFiles) + AnalyseProject(proj); + } + + void AnalyseProject(FileInfo project) + { + if(!project.Exists) { - try - { - var csProj = new CsProjFile(proj); - - foreach (var @ref in csProj.References) - { - AssemblyInfo resolved = assemblyCache.ResolveReference(@ref); - if (!resolved.Valid) - { - UnresolvedReference(@ref, proj.FullName); - } - else - { - UseReference(resolved.Filename); - } - } - - foreach (var src in csProj.Sources) - { - // Make a note of which source files the projects use. - // This information doesn't affect the build but is dumped - // as diagnostic output. - UseSource(new FileInfo(src)); - } - ++succeededProjects; - } - catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] - { - ++failedProjects; - progressMonitor.FailedProjectFile(proj.FullName, ex.Message); - } + progressMonitor.MissingProject(project.FullName); + return; } + + try + { + var csProj = new CsProjFile(project); + + foreach (var @ref in csProj.References) + { + AssemblyInfo resolved = assemblyCache.ResolveReference(@ref); + if (!resolved.Valid) + { + UnresolvedReference(@ref, project.FullName); + } + else + { + UseReference(resolved.Filename); + } + } + + foreach (var src in csProj.Sources) + { + // Make a note of which source files the projects use. + // This information doesn't affect the build but is dumped + // as diagnostic output. + UseSource(new FileInfo(src)); + } + + ++succeededProjects; + } + catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] + { + ++failedProjects; + progressMonitor.FailedProjectFile(project.FullName, ex.Message); + } + } /// @@ -296,17 +323,36 @@ namespace Semmle.BuildAnalyser /// public void Cleanup() { - if (nuget != null) nuget.Cleanup(progressMonitor); + PackageDirectory?.Cleanup(); } - /// - /// Analyse all project files in a given solution only. - /// - /// The filename of the solution. - public void AnalyseSolution(string solutionFile) + void Restore(string projectOrSolution) { - var sln = new SolutionFile(solutionFile); - AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).ToArray()); + int exit = DotNet.RestoreToDirectory(projectOrSolution, PackageDirectory.DirInfo.FullName); + if (exit != 0) + progressMonitor.CommandFailed("dotnet", $"restore \"{projectOrSolution}\"", exit); + } + + public void RestoreSolutions(IEnumerable solutions) + { + Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = 4 }, Restore); + } + + public void AnalyseSolutions(IEnumerable solutions) + { + Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = 4 } , solutionFile => + { + try + { + var sln = new SolutionFile(solutionFile); + progressMonitor.AnalysingSolution(solutionFile); + AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).Where(p => p.Exists).ToArray()); + } + catch (Microsoft.Build.Exceptions.InvalidProjectFileException ex) + { + progressMonitor.FailedProjectFile(solutionFile, ex.BaseMessage); + } + }); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs index 2c9e72c1eaa..02391c6eb7c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs @@ -5,17 +5,97 @@ using System.Xml; namespace Semmle.BuildAnalyser { + /// + /// A reference to a particular version of a particular package. + /// + class PackageReference + { + public PackageReference(string include, string version) { + Include = include; + Version = version; + } + public string Include, Version; + + public override string ToString() => $"Include={Include}, Version={Version}"; + } + + enum ProjectFileType + { + MsBuildProject, + DotNetProject, + OtherProject + } + + interface IProjectFile + { + IEnumerable ProjectReferences { get; } + + IEnumerable Packages { get; } + + IEnumerable References { get; } + + IEnumerable Sources { get; } + + IEnumerable TargetFrameworks { get; } + } + + class NetCoreProjectFile : IProjectFile + { + FileInfo path; + XmlDocument doc; + XmlElement root; + + public NetCoreProjectFile(FileInfo path) + { + this.path = path; + doc = new XmlDocument(); + doc.Load(path.FullName); + root = doc.DocumentElement; + } + + public IEnumerable ProjectReferences => throw new System.NotImplementedException(); + + public IEnumerable Packages + { + get + { + var packages = root.SelectNodes("/Project/ItemGroup/PackageReference"); + return packages.NodeList(). + Select(r => + new PackageReference(r.Attributes.GetNamedItem("Include").Value, r.Attributes.GetNamedItem("Version").Value)); + } + } + + public IEnumerable References => throw new System.NotImplementedException(); + + public IEnumerable Sources + { + get + { + return path.Directory.GetFiles("*.cs", SearchOption.AllDirectories); + } + } + + public IEnumerable TargetFrameworks => throw new System.NotImplementedException(); + } + /// /// Represents a .csproj file and reads information from it. /// class CsProjFile { + public string Filename { get; } + + public string Directory => Path.GetDirectoryName(Filename); + /// /// Reads the .csproj file. /// /// The .csproj file. public CsProjFile(FileInfo filename) { + Filename = filename.FullName; + try { // This can fail if the .csproj is invalid or has @@ -56,6 +136,8 @@ namespace Semmle.BuildAnalyser .ToArray(); } + string[] targetFrameworks = new string[0]; + /// /// Reads the .csproj file directly as XML. /// This doesn't handle variables etc, and should only used as a @@ -71,6 +153,35 @@ namespace Semmle.BuildAnalyser var projDir = filename.Directory; var root = projFile.DocumentElement; + // Figure out if it's dotnet core + + bool netCoreProjectFile = root.GetAttribute("Sdk") == "Microsoft.NET.Sdk"; + + if(netCoreProjectFile) + { + var frameworksNode = root.SelectNodes("/Project/PropertyGroup/TargetFrameworks").NodeList().Concat( + root.SelectNodes("/Project/PropertyGroup/TargetFramework").NodeList()).Select(node => node.InnerText); + + targetFrameworks = frameworksNode.SelectMany(node => node.Split(";")).ToArray(); + + var relativeCsIncludes2 = + root.SelectNodes("/Project/ItemGroup/Compile/@Include", mgr). + NodeList(). + Select(node => node.Value). + ToArray(); + + var explicitCsFiles = relativeCsIncludes2. + Select(cs => Path.DirectorySeparatorChar == '/' ? cs.Replace("\\", "/") : cs). + Select(f => Path.GetFullPath(Path.Combine(projDir.FullName, f))); + + var additionalCsFiles = System.IO.Directory.GetFiles(Directory, "*.cs", SearchOption.AllDirectories); + + csFiles = explicitCsFiles.Concat(additionalCsFiles).ToArray(); + + references = new string[0]; + return; + } + references = root.SelectNodes("/msbuild:Project/msbuild:ItemGroup/msbuild:Reference/@Include", mgr). NodeList(). @@ -97,6 +208,8 @@ namespace Semmle.BuildAnalyser /// public IEnumerable References => references; + public IEnumerable TargetFrameworks => targetFrameworks; + /// /// The list of C# source files in full path format. /// diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs new file mode 100644 index 00000000000..6d36892c7c9 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs @@ -0,0 +1,48 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; + +namespace Semmle.BuildAnalyser +{ + /// + /// Utilities to run the "dotnet" command. + /// + static class DotNet + { + public static int RestoreToDirectory(string projectOrSolutionFile, string packageDirectory) + { + using var proc = Process.Start("dotnet", $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\""); + proc.WaitForExit(); + return proc.ExitCode; + } + } + + /// + /// Utility to temporarily rename a set of files. + /// + class FileRenamer : IDisposable + { + string[] files; + const string suffix = ".codeqlhidden"; + + public FileRenamer(IEnumerable oldFiles) + { + files = oldFiles.Select(f => f.FullName).ToArray(); + + foreach(var file in files) + { + File.Move(file, file + suffix); + } + } + + public void Dispose() + { + foreach (var file in files) + { + File.Move(file + suffix, file); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNetRuntimeInfo.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNetRuntimeInfo.cs new file mode 100644 index 00000000000..21fe4a22338 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNetRuntimeInfo.cs @@ -0,0 +1,132 @@ +using System; +using System.Collections.Concurrent; +using System.Diagnostics; + +namespace RoslynWS +{ + /// + /// Information about the .NET Core runtime. + /// + public class DotNetRuntimeInfo + { + /// + /// A cache of .NET runtime information by target directory. + /// + static readonly ConcurrentDictionary _cache = new ConcurrentDictionary(); + + /// + /// The .NET Core version. + /// + public string Version { get; set; } + + /// + /// The .NET Core base directory. + /// + public string BaseDirectory { get; set; } + + /// + /// The current runtime identifier (RID). + /// + public string RID { get; set; } + + /// + /// Get information about the current .NET Core runtime. + /// + /// + /// An optional base directory where dotnet.exe should be run (this may affect the version it reports due to global.json). + /// + /// + /// A containing the runtime information. + /// + public static DotNetRuntimeInfo GetCurrent(string baseDirectory = null) + { + return _cache.GetOrAdd(baseDirectory, _ => + { + DotNetRuntimeInfo runtimeInfo = new DotNetRuntimeInfo(); + + Process dotnetInfoProcess = Process.Start(new ProcessStartInfo + { + FileName = "dotnet", + WorkingDirectory = baseDirectory, + Arguments = "--info", + UseShellExecute = false, + RedirectStandardOutput = true + }); + using (dotnetInfoProcess) + { + dotnetInfoProcess.WaitForExit(); + + string currentSection = null; + string currentLine; + while ((currentLine = dotnetInfoProcess.StandardOutput.ReadLine()) != null) + { + if (String.IsNullOrWhiteSpace(currentLine)) + continue; + + if (!currentLine.StartsWith(" ")) + { + currentSection = currentLine; + + continue; + } + + string[] property = currentLine.Split(new char[] { ':' }, count: 2); + if (property.Length != 2) + continue; + + property[0] = property[0].Trim(); + property[1] = property[1].Trim(); + + switch (currentSection) + { + case "Product Information:": + { + switch (property[0]) + { + case "Version": + { + runtimeInfo.Version = property[1]; + + break; + } + } + + break; + } + case "Runtime Environment:": + { + switch (property[0]) + { + case "Base Path": + { + runtimeInfo.BaseDirectory = property[1]; + + break; + } + case "RID": + { + runtimeInfo.RID = property[1]; + + break; + } + } + + break; + } + } + } + } + + return runtimeInfo; + }); + } + + /// + /// Clear the cache of .NET runtime information. + /// + public static void ClearCache() + { + _cache.Clear(); + } + } +} \ No newline at end of file diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/MSBuildHelper.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/MSBuildHelper.cs new file mode 100644 index 00000000000..cbba36e8419 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/MSBuildHelper.cs @@ -0,0 +1,260 @@ + +using Microsoft.Build.Construction; +using Microsoft.Build.Evaluation; +using Microsoft.Build.Exceptions; +using System; +using System.Collections.Generic; +using System.IO; +using System.Collections.Immutable; + +namespace RoslynWS +{ + /// + /// Helper methods for working with MSBuild projects. + /// + public static class MSBuildHelper + { + /// + /// The names of well-known item metadata. + /// + public static readonly ImmutableSortedSet WellknownMetadataNames = + ImmutableSortedSet.Create( + "FullPath", + "RootDir", + "Filename", + "Extension", + "RelativeDir", + "Directory", + "RecursiveDir", + "Identity", + "ModifiedTime", + "CreatedTime", + "AccessedTime" + ); + + /// + /// Create an MSBuild project collection. + /// + /// + /// The base (i.e. solution) directory. + /// + /// + /// The project collection. + /// + public static ProjectCollection CreateProjectCollection(string solutionDirectory) + { + return CreateProjectCollection(solutionDirectory, + DotNetRuntimeInfo.GetCurrent(solutionDirectory) + ); + } + + /// + /// Create an MSBuild project collection. + /// + /// + /// The base (i.e. solution) directory. + /// + /// + /// Information about the current .NET Core runtime. + /// + /// + /// The project collection. + /// + public static ProjectCollection CreateProjectCollection(string solutionDirectory, DotNetRuntimeInfo runtimeInfo) + { + if (String.IsNullOrWhiteSpace(solutionDirectory)) + throw new ArgumentException("Argument cannot be null, empty, or entirely composed of whitespace: 'baseDir'.", nameof(solutionDirectory)); + + if (runtimeInfo == null) + throw new ArgumentNullException(nameof(runtimeInfo)); + + if (String.IsNullOrWhiteSpace(runtimeInfo.BaseDirectory)) + throw new InvalidOperationException("Cannot determine base directory for .NET Core."); + + Dictionary globalProperties = CreateGlobalMSBuildProperties(runtimeInfo, solutionDirectory); + EnsureMSBuildEnvironment(globalProperties); + + ProjectCollection projectCollection = new ProjectCollection(globalProperties) { IsBuildEnabled = false }; + + // Override toolset paths (for some reason these point to the main directory where the dotnet executable lives). + Toolset toolset = projectCollection.GetToolset("15.0"); + toolset = new Toolset( + toolsVersion: "15.0", + toolsPath: globalProperties["MSBuildExtensionsPath"], + projectCollection: projectCollection, + msbuildOverrideTasksPath: "" + ); + projectCollection.AddToolset(toolset); + + return projectCollection; + } + + /// + /// Create global properties for MSBuild. + /// + /// + /// Information about the current .NET Core runtime. + /// + /// + /// The base (i.e. solution) directory. + /// + /// + /// A dictionary containing the global properties. + /// + public static Dictionary CreateGlobalMSBuildProperties(DotNetRuntimeInfo runtimeInfo, string solutionDirectory) + { + if (runtimeInfo == null) + throw new ArgumentNullException(nameof(runtimeInfo)); + + if (String.IsNullOrWhiteSpace(solutionDirectory)) + throw new ArgumentException("Argument cannot be null, empty, or entirely composed of whitespace: 'solutionDirectory'.", nameof(solutionDirectory)); + + if (solutionDirectory.Length > 0 && solutionDirectory[solutionDirectory.Length - 1] != Path.DirectorySeparatorChar) + solutionDirectory += Path.DirectorySeparatorChar; + + return new Dictionary + { + [WellKnownPropertyNames.DesignTimeBuild] = "true", + [WellKnownPropertyNames.BuildProjectReferences] = "false", + [WellKnownPropertyNames.ResolveReferenceDependencies] = "true", + [WellKnownPropertyNames.SolutionDir] = solutionDirectory, + [WellKnownPropertyNames.MSBuildExtensionsPath] = runtimeInfo.BaseDirectory, + [WellKnownPropertyNames.MSBuildSDKsPath] = Path.Combine(runtimeInfo.BaseDirectory, "Sdks"), + [WellKnownPropertyNames.RoslynTargetsPath] = Path.Combine(runtimeInfo.BaseDirectory, "Roslyn") + }; + } + + /// + /// Ensure that environment variables are populated using the specified MSBuild global properties. + /// + /// + /// The MSBuild global properties + /// + public static void EnsureMSBuildEnvironment(Dictionary globalMSBuildProperties) + { + if (globalMSBuildProperties == null) + throw new ArgumentNullException(nameof(globalMSBuildProperties)); + + // Kinda sucks that the simplest way to get MSBuild to resolve SDKs correctly is using environment variables, but there you go. + Environment.SetEnvironmentVariable( + WellKnownPropertyNames.MSBuildExtensionsPath, + globalMSBuildProperties[WellKnownPropertyNames.MSBuildExtensionsPath] + ); + Environment.SetEnvironmentVariable( + WellKnownPropertyNames.MSBuildSDKsPath, + globalMSBuildProperties[WellKnownPropertyNames.MSBuildSDKsPath] + ); + } + + /// + /// Does the specified property name represent a private property? + /// + /// + /// The property name. + /// + /// + /// true, if the property name starts with an underscore; otherwise, false. + /// + public static bool IsPrivateProperty(string propertyName) => propertyName?.StartsWith("_") ?? false; + + /// + /// Does the specified metadata name represent a private property? + /// + /// + /// The metadata name. + /// + /// + /// true, if the metadata name starts with an underscore; otherwise, false. + /// + public static bool IsPrivateMetadata(string metadataName) => metadataName?.StartsWith("_") ?? false; + + /// + /// Does the specified item type represent a private property? + /// + /// + /// The item type. + /// + /// + /// true, if the item type starts with an underscore; otherwise, false. + /// + public static bool IsPrivateItemType(string itemType) => itemType?.StartsWith("_") ?? false; + + /// + /// Determine whether the specified metadata name represents well-known (built-in) item metadata. + /// + /// + /// The metadata name. + /// + /// + /// true, if represents well-known item metadata; otherwise, false. + /// + public static bool IsWellKnownItemMetadata(string metadataName) => WellknownMetadataNames.Contains(metadataName); + + /// + /// Create a copy of the project for caching. + /// + /// + /// The MSBuild project. + /// + /// + /// The project copy (independent of original, but sharing the same ). + /// + /// + /// You can only create a single cached copy for a given project. + /// + public static Project CloneAsCachedProject(this Project project) + { + if (project == null) + throw new ArgumentNullException(nameof(project)); + + ProjectRootElement clonedXml = project.Xml.DeepClone(); + Project clonedProject = new Project(clonedXml, project.GlobalProperties, project.ToolsVersion, project.ProjectCollection); + clonedProject.FullPath = Path.ChangeExtension(project.FullPath, + ".cached" + Path.GetExtension(project.FullPath) + ); + + return clonedProject; + } + + /// + /// The names of well-known MSBuild properties. + /// + public static class WellKnownPropertyNames + { + /// + /// The "MSBuildExtensionsPath" property. + /// + public static readonly string MSBuildExtensionsPath = "MSBuildExtensionsPath"; + + /// + /// The "MSBuildSDKsPath" property. + /// + public static readonly string MSBuildSDKsPath = "MSBuildSDKsPath"; + + /// + /// The "SolutionDir" property. + /// + public static readonly string SolutionDir = "SolutionDir"; + + /// + /// The "_ResolveReferenceDependencies" property. + /// + public static readonly string ResolveReferenceDependencies = "_ResolveReferenceDependencies"; + + /// + /// The "DesignTimeBuild" property. + /// + public static readonly string DesignTimeBuild = "DesignTimeBuild"; + + /// + /// The "BuildProjectReferences" property. + /// + public static readonly string BuildProjectReferences = "BuildProjectReferences"; + + /// + /// The "RoslynTargetsPath" property. + /// + public static readonly string RoslynTargetsPath = "RoslynTargetsPath"; + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackageRepository.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackageRepository.cs new file mode 100644 index 00000000000..3b84df7463c --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackageRepository.cs @@ -0,0 +1,357 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; + +namespace Semmle.BuildAnalyser +{ + /// + /// A package in a NuGet package repository. + /// For example, the directory C:\Users\calum\.nuget\packages\microsoft.visualbasic. + /// + /// Each package contains a number of subdirectories, organised by version. + /// + class Package + { + public readonly string directory; + + /// + /// Constructs a package for the given directory. + /// + /// The directory. + public Package(string dir) + { + directory = dir; + } + + /// + /// The name of the package. + /// + public string Name => Path.GetDirectoryName(directory); + + /// + /// The versions that exist within the package. + /// + public IEnumerable Versions => Directory.GetDirectories(directory).Select(dir => new PackageVersion(dir)); + + public override string ToString() => Path.GetFileName(directory); + + public PackageVersion FindVersion(string version) + { + if (Directory.Exists(Path.Combine(directory, version))) + return new PackageVersion(Path.Combine(directory, version)); + else + return Versions.OrderByDescending(v => v.Version).First(); + } + + /// + /// Locates the exact version of a particular package. + /// + /// The version to locate. + /// The specific version of the package. + public PackageVersion FindExactVersion(string version) + { + if (Directory.Exists(Path.Combine(directory, version))) + return new PackageVersion(Path.Combine(directory, version)); + else + return null; + } + } + + /// + /// A package in a NuGet package respository, including the specific version. + /// For example, the directory C:\Users\calum\.nuget\packages\microsoft.visualbasic\10.0.1 + /// + class PackageVersion + { + readonly string directory; + + /// + /// The version of the package. + /// + public string Version => Path.GetFileName(directory); + + /// + /// Constructs a package version from its directory. + /// + /// The directory of this package. + public PackageVersion(string directory) + { + if (!Directory.Exists(directory)) + throw new DirectoryNotFoundException(directory); + + this.directory = directory; + } + + public override string ToString() => Version; + + /// + /// The frameworks within this package. + /// Sometimes a directory references several frameworks, for example + /// "net451+netstandard2.0". This are split into separate frameworks. + /// + IEnumerable UnorderedFrameworks + { + get + { + return UnorderedFrameworksInDirectory(Path.Combine(directory, "lib")). + Concat(UnorderedFrameworksInDirectory(Path.Combine(directory, "ref"))). + Concat(UnorderedFrameworksInDirectory(Path.Combine(directory, "build", "lib"))). + Concat(UnorderedFrameworksInDirectory(Path.Combine(directory, "build", "ref"))). + Concat(TryDirectory(Path.Combine(directory, "lib"))); + } + } + + IEnumerable TryDirectory(string directory) + { + if (Directory.Exists(directory)) + yield return new PackageFramework(directory, "unknown"); + } + + IEnumerable UnorderedFrameworksInDirectory(string lib) + { + if (!Directory.Exists(lib)) + yield break; + foreach (var p in System.IO.Directory.GetDirectories(lib)) + { + var name = Path.GetFileName(p); + if (name.Contains('+')) + { + foreach (var p2 in name.Split('+')) + yield return new PackageFramework(p, p2); + } + else + yield return new PackageFramework(p, name); + } + } + + /// + /// The frameworks in this package, in a consistent sequence, with the "best" frameworks + /// appearing at the start of the list. + /// + /// Priorities "netstandard" framework, followed by "netcoreapp", followed by everything else. + /// Then, selects the highest version number. + /// + public IEnumerable Frameworks => + UnorderedFrameworks. + OrderBy(framework => framework.Framework.StartsWith("netstandard") ? 0 : framework.Framework.StartsWith("netcoreapp") ? 1 : 2). + ThenByDescending(framework => framework.Framework); + + /// + /// Finds the best framework containing references. + /// Returns null if no suitable framework was found. + /// + public PackageFramework? BestFramework => Frameworks.Where(f => f.References.Any()).FirstOrDefault(); + + public bool ContainsLibraries + { + get + { + return Directory.Exists(Path.Combine(directory, "lib")) || + Directory.Exists(Path.Combine(directory, "ref")) || + Directory.Exists(Path.Combine(directory, "build", "lib")) || + Directory.Exists(Path.Combine(directory, "build", "ref")); + } + } + + public bool ContainsDLLs + { + get + { + return Directory.GetFiles(directory, "*.dll", SearchOption.AllDirectories).Any(); + } + } + } + + /// + /// A framework in a package. + /// For example, C:\Users\calum\.nuget\packages\microsoft.testplatform.objectmodel\16.4.0\lib\netstandard2.0 + /// + class PackageFramework + { + public string Directory { get; } + + /// + /// The framework name. + /// + public string Framework { get; } + + /// + /// Constructs a package framework from a directory. + /// The framework is needed because the directory may specify more than one framework. + /// + /// The directory path. + /// The framework. + public PackageFramework(string dir, string framework) + { + if (!System.IO.Directory.Exists(dir)) + throw new FileNotFoundException(dir); + Directory = dir; + Framework = framework; + } + + /// + /// The reference DLLs contained within the directory. + /// + public IEnumerable References + { + get + { + return new DirectoryInfo(Directory).GetFiles("*.dll").Select(fi => fi.FullName); + } + } + + public override string ToString() => Directory; + } + + class TemporaryDirectory : IDisposable + { + readonly IProgressMonitor ProgressMonitor; + + public DirectoryInfo DirInfo { get; } + + public TemporaryDirectory(string name, IProgressMonitor pm) + { + ProgressMonitor = pm; + DirInfo = new DirectoryInfo(name); + DirInfo.Create(); + } + + /// + /// Computes a unique temp directory for the packages associated + /// with this source tree. Use a SHA1 of the directory name. + /// + /// + /// The full path of the temp directory. + public static string ComputeTempDirectory(string srcDir) + { + var bytes = Encoding.Unicode.GetBytes(srcDir); + + var sha1 = new SHA1CryptoServiceProvider(); + var sha = sha1.ComputeHash(bytes); + var sb = new StringBuilder(); + foreach (var b in sha.Take(8)) + sb.AppendFormat("{0:x2}", b); + + return Path.Combine(Path.GetTempPath(), "GitHub", "packages", sb.ToString()); + } + + public static TemporaryDirectory CreateTempDirectory(string source, IProgressMonitor pm) => new TemporaryDirectory(ComputeTempDirectory(source), pm); + + public void Cleanup() + { + try + { + DirInfo.Delete(true); + } + catch (System.IO.IOException ex) + { + ProgressMonitor.Warning(string.Format("Couldn't delete package directory - it's probably held open by something else: {0}", ex.Message)); + } + + } + + public void Dispose() + { + Cleanup(); + } + + public override string ToString() => DirInfo.FullName.ToString(); + } + + /// + /// The NuGet package repository. + /// + class NugetPackageRepository + { + // A list of package directories, in the order they should be searched. + private readonly string[] packageDirs; + + public NugetPackageRepository(params string[] dirs) + { + packageDirs = dirs; + } + + /// + /// Constructs a NuGet package repository, using the default locations. + /// For example, + /// $HOME/.nuget/packages, /usr/share/dotnet/sdk/NuGetFallbackFolder + /// + public NugetPackageRepository(string sourceDir) + { + var homeFolder = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + var nugetPackages = Path.Combine(homeFolder, ".nuget", "packages"); + var nugetFallbackFolder = Environment.OSVersion.Platform == PlatformID.Win32NT ? + @"C:\Program Files\dotnet\sdk\NuGetFallbackFolder" : + "/usr/share/dotnet/sdk/NuGetFallbackFolder"; + + packageDirs = new string[] { nugetPackages, nugetFallbackFolder }; + } + + /// + /// Enumerate all available packages. + /// + public IEnumerable Packages + { + get + { + foreach (var d in packageDirs) + foreach (var p in Directory.GetDirectories(d)) + { + var name = Path.GetFileName(p); + if (!name.StartsWith('.')) + yield return new Package(p); + } + } + } + + /// + /// Tries to find a PackageFramework directory for a given package reference. + /// + /// The package reference to search for. + /// The package that was found. + /// True if a package/version/framework was found. + public bool TryFindLibs(PackageReference reference, out PackageFramework package, out ResolutionFailureReason reason) + { + var packages = packageDirs. + Where(d => Directory.Exists(Path.Combine(d, reference.Include.ToLowerInvariant()))). + Select(d => new Package(Path.Combine(d, reference.Include.ToLowerInvariant()))); + + if(!packages.Any()) + { + reason = ResolutionFailureReason.PackageNotFound; + package = null; + return false; + } + + var version = packages.Select(p => p.FindVersion(reference.Version)).FirstOrDefault(v => !(v is null)); + + if (version is null) + { + reason = ResolutionFailureReason.VersionNotFound; + package = null; + return false; + } + + package = version.BestFramework; + if(package is null) + { + reason = ResolutionFailureReason.LibsNotFound; + return false; + } + + reason = ResolutionFailureReason.Success; + return true; + } + } + + enum ResolutionFailureReason + { + Success, + PackageNotFound, + VersionNotFound, + LibsNotFound + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs index 1f0755f307f..0fc207f2049 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs @@ -19,10 +19,10 @@ namespace Semmle.BuildAnalyser /// Create the package manager for a specified source tree. /// /// The source directory. - public NugetPackages(string sourceDir) + public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory) { SourceDirectory = sourceDir; - PackageDirectory = computeTempDirectory(sourceDir); + PackageDirectory = packageDirectory; // Expect nuget.exe to be in a `nuget` directory under the directory containing this exe. var currentAssembly = System.Reflection.Assembly.GetExecutingAssembly().Location; @@ -50,45 +50,12 @@ namespace Semmle.BuildAnalyser /// public IEnumerable PackageFiles => packages; - // Whether to delete the packages directory prior to each run. - // Makes each build more reproducible. - const bool cleanupPackages = true; - - public void Cleanup(IProgressMonitor pm) - { - var packagesDirectory = new DirectoryInfo(PackageDirectory); - - if (packagesDirectory.Exists) - { - try - { - packagesDirectory.Delete(true); - } - catch (System.IO.IOException ex) - { - pm.Warning(string.Format("Couldn't delete package directory - it's probably held open by something else: {0}", ex.Message)); - } - } - } - /// /// Download the packages to the temp folder. /// /// The progress monitor used for reporting errors etc. public void InstallPackages(IProgressMonitor pm) { - if (cleanupPackages) - { - Cleanup(pm); - } - - var packagesDirectory = new DirectoryInfo(PackageDirectory); - - if (!Directory.Exists(PackageDirectory)) - { - packagesDirectory.Create(); - } - foreach (var package in packages) { RestoreNugetPackage(package.FullName, pm); @@ -109,31 +76,7 @@ namespace Semmle.BuildAnalyser /// This will be in the Temp location /// so as to not trample the source tree. /// - public string PackageDirectory - { - get; - private set; - } - - readonly SHA1CryptoServiceProvider sha1 = new SHA1CryptoServiceProvider(); - - /// - /// Computes a unique temp directory for the packages associated - /// with this source tree. Use a SHA1 of the directory name. - /// - /// - /// The full path of the temp directory. - string computeTempDirectory(string srcDir) - { - var bytes = Encoding.Unicode.GetBytes(srcDir); - - var sha = sha1.ComputeHash(bytes); - var sb = new StringBuilder(); - foreach (var b in sha.Take(8)) - sb.AppendFormat("{0:x2}", b); - - return Path.Combine(Path.GetTempPath(), "Semmle", "packages", sb.ToString()); - } + public TemporaryDirectory PackageDirectory { get; } /// /// Restore all files in a specified package. diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs index e0367fa63c1..3ba368f6da0 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs @@ -3,6 +3,11 @@ using System.Collections.Generic; using System.Linq; using Semmle.BuildAnalyser; using Semmle.Util.Logging; +using System.IO; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Text; + +// using Microsoft.Build.Locator; namespace Semmle.Extraction.CSharp.Standalone { @@ -82,9 +87,15 @@ namespace Semmle.Extraction.CSharp.Standalone public class Program { + void LoadSolutionFile(string file) + { + + } + static int Main(string[] args) { var options = Options.Create(args); + options.CIL = true; var output = new ConsoleLogger(options.Verbosity); var a = new Analysis(output); @@ -97,6 +108,8 @@ namespace Semmle.Extraction.CSharp.Standalone if (options.Errors) return 1; + var start = DateTime.Now; + output.Log(Severity.Info, "Running C# standalone extractor"); a.AnalyseProjects(options); int sourceFiles = a.Extraction.Sources.Count(); @@ -117,7 +130,7 @@ namespace Semmle.Extraction.CSharp.Standalone new ExtractionProgress(output), new FileLogger(options.Verbosity, Extractor.GetCSharpLogPath()), options); - output.Log(Severity.Info, "Extraction complete"); + output.Log(Severity.Info, $"Extraction completed in {DateTime.Now-start}"); } a.Cleanup(); @@ -151,7 +164,7 @@ namespace Semmle.Extraction.CSharp.Standalone public void MissingSummary(int missingTypes, int missingNamespaces) { - logger.Log(Severity.Info, "Failed to resolve {0} types and {1} namespaces", missingTypes, missingNamespaces); + logger.Log(Severity.Info, "Failed to resolve {0} types in {1} namespaces", missingTypes, missingNamespaces); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs index f4bde55ec55..a004e59eb13 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs @@ -10,6 +10,7 @@ namespace Semmle.BuildAnalyser void FindingFiles(string dir); void UnresolvedReference(string id, string project); void AnalysingProjectFiles(int count); + void AnalysingSolution(string filename); void FailedProjectFile(string filename, string reason); void FailedNugetCommand(string exe, string args, string message); void NugetInstall(string package); @@ -18,6 +19,11 @@ namespace Semmle.BuildAnalyser void Warning(string message); void ResolvedConflict(string asm1, string asm2); void MissingProject(string projectFile); + void Restored(string line); + void MissingPackage(string package, string version, ResolutionFailureReason reason); + void FoundPackage(string package, string version, string directory); + void CommandFailed(string exe, string arguments, int exitCode); + void MissingNuGet(); } class ProgressMonitor : IProgressMonitor @@ -51,6 +57,11 @@ namespace Semmle.BuildAnalyser logger.Log(Severity.Info, "Analyzing project files..."); } + public void AnalysingSolution(string filename) + { + logger.Log(Severity.Info, $"Analysing {filename}..."); + } + public void FailedProjectFile(string filename, string reason) { logger.Log(Severity.Info, "Couldn't read project file {0}: {1}", filename, reason); @@ -73,7 +84,8 @@ namespace Semmle.BuildAnalyser } public void Summary(int existingSources, int usedSources, int missingSources, - int references, int unresolvedReferences, int resolvedConflicts, int totalProjects, int failedProjects) + int references, int unresolvedReferences, + int resolvedConflicts, int totalProjects, int failedProjects) { logger.Log(Severity.Info, ""); logger.Log(Severity.Info, "Build analysis summary:"); @@ -87,6 +99,23 @@ namespace Semmle.BuildAnalyser logger.Log(Severity.Info, "{0, 6} missing/failed projects", failedProjects); } + public void Restored(string line) + { + logger.Log(Severity.Debug, $" {line}"); + } + + private static string[] reasonText = { "success", "package was not found", "the version was not found", "the package does not appear to contain any libraries" }; + + public void MissingPackage(string package, string version, ResolutionFailureReason reason) + { + logger.Log(Severity.Info, $" Couldn't find package {package} {version} because {reasonText[(int)reason]}"); + } + + public void FoundPackage(string package, string version, string directory) + { + logger.Log(Severity.Debug, $" Found package {package} {version} in {directory}"); + } + public void Warning(string message) { logger.Log(Severity.Warning, message); @@ -94,12 +123,22 @@ namespace Semmle.BuildAnalyser public void ResolvedConflict(string asm1, string asm2) { - logger.Log(Severity.Info, "Resolved {0} as {1}", asm1, asm2); + logger.Log(Severity.Debug, "Resolved {0} as {1}", asm1, asm2); } public void MissingProject(string projectFile) { logger.Log(Severity.Info, "Solution is missing {0}", projectFile); } + + public void CommandFailed(string exe, string arguments, int exitCode) + { + logger.Log(Severity.Error, $"Command {exe} {arguments} failed with exit code {exitCode}"); + } + + public void MissingNuGet() + { + logger.Log(Severity.Error, "Missing nuget.exe"); + } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj index 4cf0274b737..1d4ab57d2d8 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj @@ -1,4 +1,4 @@ - + Exe @@ -21,7 +21,10 @@ - + + + + diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/SolutionFile.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/SolutionFile.cs index b1a3edd4cf6..ffaebe360fe 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/SolutionFile.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/SolutionFile.cs @@ -11,6 +11,7 @@ namespace Semmle.BuildAnalyser class SolutionFile { readonly Microsoft.Build.Construction.SolutionFile solutionFile; + public string FullPath { get; } /// /// Read the file. @@ -19,8 +20,8 @@ namespace Semmle.BuildAnalyser public SolutionFile(string filename) { // SolutionFile.Parse() expects a rooted path. - var fullPath = Path.GetFullPath(filename); - solutionFile = Microsoft.Build.Construction.SolutionFile.Parse(fullPath); + FullPath = Path.GetFullPath(filename); + solutionFile = Microsoft.Build.Construction.SolutionFile.Parse(FullPath); } /// diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Access.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Access.cs index 0488fe84ffe..6962e8381d9 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Access.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Access.cs @@ -45,7 +45,10 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions Access(ExpressionNodeInfo info, ISymbol symbol, bool implicitThis, IEntity target) : base(info.SetKind(AccessKind(info.Context, symbol))) { - cx.TrapWriter.Writer.expr_access(this, target); + if (!(target is null)) + { + cx.TrapWriter.Writer.expr_access(this, target); + } if (implicitThis && !symbol.IsStatic) { diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MemberAccess.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MemberAccess.cs index e41ef0edf23..0bc84ca9c0c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MemberAccess.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MemberAccess.cs @@ -71,7 +71,9 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions if (symbol == null) { info.Context.ModelError(info.Node, "Failed to determine symbol for member access"); - return new MemberAccess(info.SetKind(ExprKind.UNKNOWN), expression, symbol); + // Default to property access - this can still give useful results but + // the target of the expression should be checked in QL. + return new MemberAccess(info.SetKind(ExprKind.PROPERTY_ACCESS), expression, symbol); } ExprKind kind; diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs index e22d32c0d01..cecec5bc028 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs @@ -25,7 +25,7 @@ namespace Semmle.Extraction.CSharp.Entities { if (symbol.TypeKind == TypeKind.Error) { - Context.Extractor.MissingType(symbol.ToString()); + Context.Extractor.MissingType(symbol.ToString(), Context.FromSource); return; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs index 4fdbe7c18ad..02b67efc164 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs @@ -32,7 +32,7 @@ namespace Semmle.Extraction.CSharp.Entities if (namespaceSymbol == null) { - cx.Extractor.MissingNamespace(Node.Name.ToFullString()); + cx.Extractor.MissingNamespace(Node.Name.ToFullString(), cx.FromSource); cx.ModelError(Node, "Namespace not found"); return; } diff --git a/csharp/extractor/Semmle.Extraction/Context.cs b/csharp/extractor/Semmle.Extraction/Context.cs index 93f99381858..918642f198d 100644 --- a/csharp/extractor/Semmle.Extraction/Context.cs +++ b/csharp/extractor/Semmle.Extraction/Context.cs @@ -155,7 +155,7 @@ namespace Semmle.Extraction #if DEBUG_LABELS using (var id = new StringWriter()) { - entity.WriteId(id); + entity.WriteQuotedId(id); CheckEntityHasUniqueLabel(id.ToString(), entity); } #endif @@ -270,6 +270,8 @@ namespace Semmle.Extraction TrapWriter = trapWriter; } + public bool FromSource => Scope.FromSource; + public bool IsGlobalContext => Scope.IsGlobalScope; public readonly ICommentGenerator CommentGenerator = new CommentProcessor(); diff --git a/csharp/extractor/Semmle.Extraction/ExtractionScope.cs b/csharp/extractor/Semmle.Extraction/ExtractionScope.cs index 7f4f599fe5c..60daff8d013 100644 --- a/csharp/extractor/Semmle.Extraction/ExtractionScope.cs +++ b/csharp/extractor/Semmle.Extraction/ExtractionScope.cs @@ -25,6 +25,8 @@ namespace Semmle.Extraction bool InFileScope(string path); bool IsGlobalScope { get; } + + bool FromSource { get; } } /// @@ -49,6 +51,8 @@ namespace Semmle.Extraction public bool InScope(ISymbol symbol) => SymbolEqualityComparer.Default.Equals(symbol.ContainingAssembly, assembly) || SymbolEqualityComparer.Default.Equals(symbol, assembly); + + public bool FromSource => false; } /// @@ -68,5 +72,7 @@ namespace Semmle.Extraction public bool InFileScope(string path) => path == sourceTree.FilePath; public bool InScope(ISymbol symbol) => symbol.Locations.Any(loc => loc.SourceTree == sourceTree); + + public bool FromSource => true; } } diff --git a/csharp/extractor/Semmle.Extraction/Extractor.cs b/csharp/extractor/Semmle.Extraction/Extractor.cs index e470d3258ec..e1ca23f645b 100644 --- a/csharp/extractor/Semmle.Extraction/Extractor.cs +++ b/csharp/extractor/Semmle.Extraction/Extractor.cs @@ -50,13 +50,15 @@ namespace Semmle.Extraction /// Record a new error type. /// /// The display name of the type, qualified where possible. - void MissingType(string fqn); + /// The missing type was referenced from a source file. + void MissingType(string fqn, bool fromSource); /// /// Record an unresolved `using namespace` directive. /// /// The full name of the namespace. - void MissingNamespace(string fqn); + /// The missing namespace was referenced from a source file. + void MissingNamespace(string fqn, bool fromSource); /// /// The list of missing types. @@ -167,16 +169,22 @@ namespace Semmle.Extraction readonly ISet missingTypes = new SortedSet(); readonly ISet missingNamespaces = new SortedSet(); - public void MissingType(string fqn) + public void MissingType(string fqn, bool fromSource) { - lock (mutex) - missingTypes.Add(fqn); + if (fromSource) + { + lock (mutex) + missingTypes.Add(fqn); + } } - public void MissingNamespace(string fqdn) + public void MissingNamespace(string fqdn, bool fromSource) { - lock (mutex) - missingNamespaces.Add(fqdn); + if (fromSource) + { + lock (mutex) + missingNamespaces.Add(fqdn); + } } public Context CreateContext(Compilation c, TrapWriter trapWriter, IExtractionScope scope) From 71e0dc087b7da56b0d10c373332c72726760795e Mon Sep 17 00:00:00 2001 From: Calum Grant Date: Thu, 26 Mar 2020 15:35:31 +0000 Subject: [PATCH 2/5] C#: General code tidy. --- .../BuildAnalysis.cs | 55 ++- .../CsProjFile.cs | 77 +--- .../DotNet.cs | 2 +- .../DotNetRuntimeInfo.cs | 132 ------- .../MSBuildHelper.cs | 260 ------------- .../NugetPackageRepository.cs | 357 ------------------ .../NugetPackages.cs | 45 +++ .../Program.cs | 10 - .../ProgressMonitor.cs | 33 +- ...Semmle.Extraction.CSharp.Standalone.csproj | 5 +- .../SolutionFile.cs | 3 +- .../Entities/NamespaceDeclaration.cs | 3 +- .../extractor/Semmle.Extraction/Extractor.cs | 4 +- 13 files changed, 87 insertions(+), 899 deletions(-) delete mode 100644 csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNetRuntimeInfo.cs delete mode 100644 csharp/extractor/Semmle.Extraction.CSharp.Standalone/MSBuildHelper.cs delete mode 100644 csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackageRepository.cs diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs index dfc110c0d03..5ec150e5af3 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs @@ -2,11 +2,8 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Runtime.InteropServices; -using Semmle.Util; using Semmle.Extraction.CSharp.Standalone; using System.Threading.Tasks; -using System.Collections.Concurrent; namespace Semmle.BuildAnalyser { @@ -47,18 +44,18 @@ namespace Semmle.BuildAnalyser /// class BuildAnalysis : IBuildAnalysis { - readonly AssemblyCache assemblyCache; - readonly NugetPackages nuget; - readonly IProgressMonitor progressMonitor; - HashSet usedReferences = new HashSet(); - readonly HashSet usedSources = new HashSet(); - readonly HashSet missingSources = new HashSet(); - readonly Dictionary unresolvedReferences = new Dictionary(); - readonly DirectoryInfo sourceDir; - int failedProjects, succeededProjects; - readonly string[] allSources; - int conflictedReferences = 0; - object mutex = new object(); + private readonly AssemblyCache assemblyCache; + private readonly NugetPackages nuget; + private readonly IProgressMonitor progressMonitor; + private HashSet usedReferences = new HashSet(); + private readonly HashSet usedSources = new HashSet(); + private readonly HashSet missingSources = new HashSet(); + private readonly Dictionary unresolvedReferences = new Dictionary(); + private readonly DirectoryInfo sourceDir; + private int failedProjects, succeededProjects; + private readonly string[] allSources; + private int conflictedReferences = 0; + private readonly object mutex = new object(); /// /// Performs a C# build analysis. @@ -80,7 +77,7 @@ namespace Semmle.BuildAnalyser ToArray(); var dllDirNames = options.DllDirs.Select(Path.GetFullPath).ToList(); - PackageDirectory = TemporaryDirectory.CreateTempDirectory(sourceDir.FullName, progressMonitor); + PackageDirectory = TemporaryDirectory.CreateTempDirectory(sourceDir.FullName); if (options.UseNuGet) { @@ -102,6 +99,7 @@ namespace Semmle.BuildAnalyser } { + // These files can sometimes prevent `dotnet restore` from working correctly. using var renamer1 = new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories)); using var renamer2 = new FileRenamer(sourceDir.GetFiles("Directory.Build.props", SearchOption.AllDirectories)); @@ -118,12 +116,6 @@ namespace Semmle.BuildAnalyser usedReferences = new HashSet(assemblyCache.AllAssemblies.Select(a => a.Filename)); } - if (!options.AnalyseCsProjFiles) - { - usedReferences = new HashSet(assemblyCache.AllAssemblies.Select(a => a.Filename)); - } - - ResolveConflicts(); if (options.UseMscorlib) @@ -150,9 +142,8 @@ namespace Semmle.BuildAnalyser UnresolvedReferences.Count(), conflictedReferences, succeededProjects + failedProjects, - failedProjects); - - Console.WriteLine($"Build analysis completed in {DateTime.Now - startTime}"); + failedProjects, + DateTime.Now - startTime); } /// @@ -285,7 +276,7 @@ namespace Semmle.BuildAnalyser try { - var csProj = new CsProjFile(project); + IProjectFile csProj = new CsProjFile(project); foreach (var @ref in csProj.References) { @@ -329,8 +320,16 @@ namespace Semmle.BuildAnalyser void Restore(string projectOrSolution) { int exit = DotNet.RestoreToDirectory(projectOrSolution, PackageDirectory.DirInfo.FullName); - if (exit != 0) - progressMonitor.CommandFailed("dotnet", $"restore \"{projectOrSolution}\"", exit); + switch(exit) + { + case 0: + case 1: + // No errors + break; + default: + progressMonitor.CommandFailed("dotnet", $"restore \"{projectOrSolution}\"", exit); + break; + } } public void RestoreSolutions(IEnumerable solutions) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs index 02391c6eb7c..7394eddb39c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs @@ -5,84 +5,17 @@ using System.Xml; namespace Semmle.BuildAnalyser { - /// - /// A reference to a particular version of a particular package. - /// - class PackageReference - { - public PackageReference(string include, string version) { - Include = include; - Version = version; - } - public string Include, Version; - - public override string ToString() => $"Include={Include}, Version={Version}"; - } - - enum ProjectFileType - { - MsBuildProject, - DotNetProject, - OtherProject - } - interface IProjectFile { - IEnumerable ProjectReferences { get; } - - IEnumerable Packages { get; } - IEnumerable References { get; } - IEnumerable Sources { get; } - - IEnumerable TargetFrameworks { get; } - } - - class NetCoreProjectFile : IProjectFile - { - FileInfo path; - XmlDocument doc; - XmlElement root; - - public NetCoreProjectFile(FileInfo path) - { - this.path = path; - doc = new XmlDocument(); - doc.Load(path.FullName); - root = doc.DocumentElement; - } - - public IEnumerable ProjectReferences => throw new System.NotImplementedException(); - - public IEnumerable Packages - { - get - { - var packages = root.SelectNodes("/Project/ItemGroup/PackageReference"); - return packages.NodeList(). - Select(r => - new PackageReference(r.Attributes.GetNamedItem("Include").Value, r.Attributes.GetNamedItem("Version").Value)); - } - } - - public IEnumerable References => throw new System.NotImplementedException(); - - public IEnumerable Sources - { - get - { - return path.Directory.GetFiles("*.cs", SearchOption.AllDirectories); - } - } - - public IEnumerable TargetFrameworks => throw new System.NotImplementedException(); + IEnumerable Sources { get; } } /// /// Represents a .csproj file and reads information from it. /// - class CsProjFile + class CsProjFile : IProjectFile { public string Filename { get; } @@ -136,8 +69,6 @@ namespace Semmle.BuildAnalyser .ToArray(); } - string[] targetFrameworks = new string[0]; - /// /// Reads the .csproj file directly as XML. /// This doesn't handle variables etc, and should only used as a @@ -162,8 +93,6 @@ namespace Semmle.BuildAnalyser var frameworksNode = root.SelectNodes("/Project/PropertyGroup/TargetFrameworks").NodeList().Concat( root.SelectNodes("/Project/PropertyGroup/TargetFramework").NodeList()).Select(node => node.InnerText); - targetFrameworks = frameworksNode.SelectMany(node => node.Split(";")).ToArray(); - var relativeCsIncludes2 = root.SelectNodes("/Project/ItemGroup/Compile/@Include", mgr). NodeList(). @@ -208,8 +137,6 @@ namespace Semmle.BuildAnalyser /// public IEnumerable References => references; - public IEnumerable TargetFrameworks => targetFrameworks; - /// /// The list of C# source files in full path format. /// diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs index 6d36892c7c9..4aad28068b0 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs @@ -22,7 +22,7 @@ namespace Semmle.BuildAnalyser /// /// Utility to temporarily rename a set of files. /// - class FileRenamer : IDisposable + sealed class FileRenamer : IDisposable { string[] files; const string suffix = ".codeqlhidden"; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNetRuntimeInfo.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNetRuntimeInfo.cs deleted file mode 100644 index 21fe4a22338..00000000000 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNetRuntimeInfo.cs +++ /dev/null @@ -1,132 +0,0 @@ -using System; -using System.Collections.Concurrent; -using System.Diagnostics; - -namespace RoslynWS -{ - /// - /// Information about the .NET Core runtime. - /// - public class DotNetRuntimeInfo - { - /// - /// A cache of .NET runtime information by target directory. - /// - static readonly ConcurrentDictionary _cache = new ConcurrentDictionary(); - - /// - /// The .NET Core version. - /// - public string Version { get; set; } - - /// - /// The .NET Core base directory. - /// - public string BaseDirectory { get; set; } - - /// - /// The current runtime identifier (RID). - /// - public string RID { get; set; } - - /// - /// Get information about the current .NET Core runtime. - /// - /// - /// An optional base directory where dotnet.exe should be run (this may affect the version it reports due to global.json). - /// - /// - /// A containing the runtime information. - /// - public static DotNetRuntimeInfo GetCurrent(string baseDirectory = null) - { - return _cache.GetOrAdd(baseDirectory, _ => - { - DotNetRuntimeInfo runtimeInfo = new DotNetRuntimeInfo(); - - Process dotnetInfoProcess = Process.Start(new ProcessStartInfo - { - FileName = "dotnet", - WorkingDirectory = baseDirectory, - Arguments = "--info", - UseShellExecute = false, - RedirectStandardOutput = true - }); - using (dotnetInfoProcess) - { - dotnetInfoProcess.WaitForExit(); - - string currentSection = null; - string currentLine; - while ((currentLine = dotnetInfoProcess.StandardOutput.ReadLine()) != null) - { - if (String.IsNullOrWhiteSpace(currentLine)) - continue; - - if (!currentLine.StartsWith(" ")) - { - currentSection = currentLine; - - continue; - } - - string[] property = currentLine.Split(new char[] { ':' }, count: 2); - if (property.Length != 2) - continue; - - property[0] = property[0].Trim(); - property[1] = property[1].Trim(); - - switch (currentSection) - { - case "Product Information:": - { - switch (property[0]) - { - case "Version": - { - runtimeInfo.Version = property[1]; - - break; - } - } - - break; - } - case "Runtime Environment:": - { - switch (property[0]) - { - case "Base Path": - { - runtimeInfo.BaseDirectory = property[1]; - - break; - } - case "RID": - { - runtimeInfo.RID = property[1]; - - break; - } - } - - break; - } - } - } - } - - return runtimeInfo; - }); - } - - /// - /// Clear the cache of .NET runtime information. - /// - public static void ClearCache() - { - _cache.Clear(); - } - } -} \ No newline at end of file diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/MSBuildHelper.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/MSBuildHelper.cs deleted file mode 100644 index cbba36e8419..00000000000 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/MSBuildHelper.cs +++ /dev/null @@ -1,260 +0,0 @@ - -using Microsoft.Build.Construction; -using Microsoft.Build.Evaluation; -using Microsoft.Build.Exceptions; -using System; -using System.Collections.Generic; -using System.IO; -using System.Collections.Immutable; - -namespace RoslynWS -{ - /// - /// Helper methods for working with MSBuild projects. - /// - public static class MSBuildHelper - { - /// - /// The names of well-known item metadata. - /// - public static readonly ImmutableSortedSet WellknownMetadataNames = - ImmutableSortedSet.Create( - "FullPath", - "RootDir", - "Filename", - "Extension", - "RelativeDir", - "Directory", - "RecursiveDir", - "Identity", - "ModifiedTime", - "CreatedTime", - "AccessedTime" - ); - - /// - /// Create an MSBuild project collection. - /// - /// - /// The base (i.e. solution) directory. - /// - /// - /// The project collection. - /// - public static ProjectCollection CreateProjectCollection(string solutionDirectory) - { - return CreateProjectCollection(solutionDirectory, - DotNetRuntimeInfo.GetCurrent(solutionDirectory) - ); - } - - /// - /// Create an MSBuild project collection. - /// - /// - /// The base (i.e. solution) directory. - /// - /// - /// Information about the current .NET Core runtime. - /// - /// - /// The project collection. - /// - public static ProjectCollection CreateProjectCollection(string solutionDirectory, DotNetRuntimeInfo runtimeInfo) - { - if (String.IsNullOrWhiteSpace(solutionDirectory)) - throw new ArgumentException("Argument cannot be null, empty, or entirely composed of whitespace: 'baseDir'.", nameof(solutionDirectory)); - - if (runtimeInfo == null) - throw new ArgumentNullException(nameof(runtimeInfo)); - - if (String.IsNullOrWhiteSpace(runtimeInfo.BaseDirectory)) - throw new InvalidOperationException("Cannot determine base directory for .NET Core."); - - Dictionary globalProperties = CreateGlobalMSBuildProperties(runtimeInfo, solutionDirectory); - EnsureMSBuildEnvironment(globalProperties); - - ProjectCollection projectCollection = new ProjectCollection(globalProperties) { IsBuildEnabled = false }; - - // Override toolset paths (for some reason these point to the main directory where the dotnet executable lives). - Toolset toolset = projectCollection.GetToolset("15.0"); - toolset = new Toolset( - toolsVersion: "15.0", - toolsPath: globalProperties["MSBuildExtensionsPath"], - projectCollection: projectCollection, - msbuildOverrideTasksPath: "" - ); - projectCollection.AddToolset(toolset); - - return projectCollection; - } - - /// - /// Create global properties for MSBuild. - /// - /// - /// Information about the current .NET Core runtime. - /// - /// - /// The base (i.e. solution) directory. - /// - /// - /// A dictionary containing the global properties. - /// - public static Dictionary CreateGlobalMSBuildProperties(DotNetRuntimeInfo runtimeInfo, string solutionDirectory) - { - if (runtimeInfo == null) - throw new ArgumentNullException(nameof(runtimeInfo)); - - if (String.IsNullOrWhiteSpace(solutionDirectory)) - throw new ArgumentException("Argument cannot be null, empty, or entirely composed of whitespace: 'solutionDirectory'.", nameof(solutionDirectory)); - - if (solutionDirectory.Length > 0 && solutionDirectory[solutionDirectory.Length - 1] != Path.DirectorySeparatorChar) - solutionDirectory += Path.DirectorySeparatorChar; - - return new Dictionary - { - [WellKnownPropertyNames.DesignTimeBuild] = "true", - [WellKnownPropertyNames.BuildProjectReferences] = "false", - [WellKnownPropertyNames.ResolveReferenceDependencies] = "true", - [WellKnownPropertyNames.SolutionDir] = solutionDirectory, - [WellKnownPropertyNames.MSBuildExtensionsPath] = runtimeInfo.BaseDirectory, - [WellKnownPropertyNames.MSBuildSDKsPath] = Path.Combine(runtimeInfo.BaseDirectory, "Sdks"), - [WellKnownPropertyNames.RoslynTargetsPath] = Path.Combine(runtimeInfo.BaseDirectory, "Roslyn") - }; - } - - /// - /// Ensure that environment variables are populated using the specified MSBuild global properties. - /// - /// - /// The MSBuild global properties - /// - public static void EnsureMSBuildEnvironment(Dictionary globalMSBuildProperties) - { - if (globalMSBuildProperties == null) - throw new ArgumentNullException(nameof(globalMSBuildProperties)); - - // Kinda sucks that the simplest way to get MSBuild to resolve SDKs correctly is using environment variables, but there you go. - Environment.SetEnvironmentVariable( - WellKnownPropertyNames.MSBuildExtensionsPath, - globalMSBuildProperties[WellKnownPropertyNames.MSBuildExtensionsPath] - ); - Environment.SetEnvironmentVariable( - WellKnownPropertyNames.MSBuildSDKsPath, - globalMSBuildProperties[WellKnownPropertyNames.MSBuildSDKsPath] - ); - } - - /// - /// Does the specified property name represent a private property? - /// - /// - /// The property name. - /// - /// - /// true, if the property name starts with an underscore; otherwise, false. - /// - public static bool IsPrivateProperty(string propertyName) => propertyName?.StartsWith("_") ?? false; - - /// - /// Does the specified metadata name represent a private property? - /// - /// - /// The metadata name. - /// - /// - /// true, if the metadata name starts with an underscore; otherwise, false. - /// - public static bool IsPrivateMetadata(string metadataName) => metadataName?.StartsWith("_") ?? false; - - /// - /// Does the specified item type represent a private property? - /// - /// - /// The item type. - /// - /// - /// true, if the item type starts with an underscore; otherwise, false. - /// - public static bool IsPrivateItemType(string itemType) => itemType?.StartsWith("_") ?? false; - - /// - /// Determine whether the specified metadata name represents well-known (built-in) item metadata. - /// - /// - /// The metadata name. - /// - /// - /// true, if represents well-known item metadata; otherwise, false. - /// - public static bool IsWellKnownItemMetadata(string metadataName) => WellknownMetadataNames.Contains(metadataName); - - /// - /// Create a copy of the project for caching. - /// - /// - /// The MSBuild project. - /// - /// - /// The project copy (independent of original, but sharing the same ). - /// - /// - /// You can only create a single cached copy for a given project. - /// - public static Project CloneAsCachedProject(this Project project) - { - if (project == null) - throw new ArgumentNullException(nameof(project)); - - ProjectRootElement clonedXml = project.Xml.DeepClone(); - Project clonedProject = new Project(clonedXml, project.GlobalProperties, project.ToolsVersion, project.ProjectCollection); - clonedProject.FullPath = Path.ChangeExtension(project.FullPath, - ".cached" + Path.GetExtension(project.FullPath) - ); - - return clonedProject; - } - - /// - /// The names of well-known MSBuild properties. - /// - public static class WellKnownPropertyNames - { - /// - /// The "MSBuildExtensionsPath" property. - /// - public static readonly string MSBuildExtensionsPath = "MSBuildExtensionsPath"; - - /// - /// The "MSBuildSDKsPath" property. - /// - public static readonly string MSBuildSDKsPath = "MSBuildSDKsPath"; - - /// - /// The "SolutionDir" property. - /// - public static readonly string SolutionDir = "SolutionDir"; - - /// - /// The "_ResolveReferenceDependencies" property. - /// - public static readonly string ResolveReferenceDependencies = "_ResolveReferenceDependencies"; - - /// - /// The "DesignTimeBuild" property. - /// - public static readonly string DesignTimeBuild = "DesignTimeBuild"; - - /// - /// The "BuildProjectReferences" property. - /// - public static readonly string BuildProjectReferences = "BuildProjectReferences"; - - /// - /// The "RoslynTargetsPath" property. - /// - public static readonly string RoslynTargetsPath = "RoslynTargetsPath"; - } - } -} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackageRepository.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackageRepository.cs deleted file mode 100644 index 3b84df7463c..00000000000 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackageRepository.cs +++ /dev/null @@ -1,357 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Security.Cryptography; -using System.Text; - -namespace Semmle.BuildAnalyser -{ - /// - /// A package in a NuGet package repository. - /// For example, the directory C:\Users\calum\.nuget\packages\microsoft.visualbasic. - /// - /// Each package contains a number of subdirectories, organised by version. - /// - class Package - { - public readonly string directory; - - /// - /// Constructs a package for the given directory. - /// - /// The directory. - public Package(string dir) - { - directory = dir; - } - - /// - /// The name of the package. - /// - public string Name => Path.GetDirectoryName(directory); - - /// - /// The versions that exist within the package. - /// - public IEnumerable Versions => Directory.GetDirectories(directory).Select(dir => new PackageVersion(dir)); - - public override string ToString() => Path.GetFileName(directory); - - public PackageVersion FindVersion(string version) - { - if (Directory.Exists(Path.Combine(directory, version))) - return new PackageVersion(Path.Combine(directory, version)); - else - return Versions.OrderByDescending(v => v.Version).First(); - } - - /// - /// Locates the exact version of a particular package. - /// - /// The version to locate. - /// The specific version of the package. - public PackageVersion FindExactVersion(string version) - { - if (Directory.Exists(Path.Combine(directory, version))) - return new PackageVersion(Path.Combine(directory, version)); - else - return null; - } - } - - /// - /// A package in a NuGet package respository, including the specific version. - /// For example, the directory C:\Users\calum\.nuget\packages\microsoft.visualbasic\10.0.1 - /// - class PackageVersion - { - readonly string directory; - - /// - /// The version of the package. - /// - public string Version => Path.GetFileName(directory); - - /// - /// Constructs a package version from its directory. - /// - /// The directory of this package. - public PackageVersion(string directory) - { - if (!Directory.Exists(directory)) - throw new DirectoryNotFoundException(directory); - - this.directory = directory; - } - - public override string ToString() => Version; - - /// - /// The frameworks within this package. - /// Sometimes a directory references several frameworks, for example - /// "net451+netstandard2.0". This are split into separate frameworks. - /// - IEnumerable UnorderedFrameworks - { - get - { - return UnorderedFrameworksInDirectory(Path.Combine(directory, "lib")). - Concat(UnorderedFrameworksInDirectory(Path.Combine(directory, "ref"))). - Concat(UnorderedFrameworksInDirectory(Path.Combine(directory, "build", "lib"))). - Concat(UnorderedFrameworksInDirectory(Path.Combine(directory, "build", "ref"))). - Concat(TryDirectory(Path.Combine(directory, "lib"))); - } - } - - IEnumerable TryDirectory(string directory) - { - if (Directory.Exists(directory)) - yield return new PackageFramework(directory, "unknown"); - } - - IEnumerable UnorderedFrameworksInDirectory(string lib) - { - if (!Directory.Exists(lib)) - yield break; - foreach (var p in System.IO.Directory.GetDirectories(lib)) - { - var name = Path.GetFileName(p); - if (name.Contains('+')) - { - foreach (var p2 in name.Split('+')) - yield return new PackageFramework(p, p2); - } - else - yield return new PackageFramework(p, name); - } - } - - /// - /// The frameworks in this package, in a consistent sequence, with the "best" frameworks - /// appearing at the start of the list. - /// - /// Priorities "netstandard" framework, followed by "netcoreapp", followed by everything else. - /// Then, selects the highest version number. - /// - public IEnumerable Frameworks => - UnorderedFrameworks. - OrderBy(framework => framework.Framework.StartsWith("netstandard") ? 0 : framework.Framework.StartsWith("netcoreapp") ? 1 : 2). - ThenByDescending(framework => framework.Framework); - - /// - /// Finds the best framework containing references. - /// Returns null if no suitable framework was found. - /// - public PackageFramework? BestFramework => Frameworks.Where(f => f.References.Any()).FirstOrDefault(); - - public bool ContainsLibraries - { - get - { - return Directory.Exists(Path.Combine(directory, "lib")) || - Directory.Exists(Path.Combine(directory, "ref")) || - Directory.Exists(Path.Combine(directory, "build", "lib")) || - Directory.Exists(Path.Combine(directory, "build", "ref")); - } - } - - public bool ContainsDLLs - { - get - { - return Directory.GetFiles(directory, "*.dll", SearchOption.AllDirectories).Any(); - } - } - } - - /// - /// A framework in a package. - /// For example, C:\Users\calum\.nuget\packages\microsoft.testplatform.objectmodel\16.4.0\lib\netstandard2.0 - /// - class PackageFramework - { - public string Directory { get; } - - /// - /// The framework name. - /// - public string Framework { get; } - - /// - /// Constructs a package framework from a directory. - /// The framework is needed because the directory may specify more than one framework. - /// - /// The directory path. - /// The framework. - public PackageFramework(string dir, string framework) - { - if (!System.IO.Directory.Exists(dir)) - throw new FileNotFoundException(dir); - Directory = dir; - Framework = framework; - } - - /// - /// The reference DLLs contained within the directory. - /// - public IEnumerable References - { - get - { - return new DirectoryInfo(Directory).GetFiles("*.dll").Select(fi => fi.FullName); - } - } - - public override string ToString() => Directory; - } - - class TemporaryDirectory : IDisposable - { - readonly IProgressMonitor ProgressMonitor; - - public DirectoryInfo DirInfo { get; } - - public TemporaryDirectory(string name, IProgressMonitor pm) - { - ProgressMonitor = pm; - DirInfo = new DirectoryInfo(name); - DirInfo.Create(); - } - - /// - /// Computes a unique temp directory for the packages associated - /// with this source tree. Use a SHA1 of the directory name. - /// - /// - /// The full path of the temp directory. - public static string ComputeTempDirectory(string srcDir) - { - var bytes = Encoding.Unicode.GetBytes(srcDir); - - var sha1 = new SHA1CryptoServiceProvider(); - var sha = sha1.ComputeHash(bytes); - var sb = new StringBuilder(); - foreach (var b in sha.Take(8)) - sb.AppendFormat("{0:x2}", b); - - return Path.Combine(Path.GetTempPath(), "GitHub", "packages", sb.ToString()); - } - - public static TemporaryDirectory CreateTempDirectory(string source, IProgressMonitor pm) => new TemporaryDirectory(ComputeTempDirectory(source), pm); - - public void Cleanup() - { - try - { - DirInfo.Delete(true); - } - catch (System.IO.IOException ex) - { - ProgressMonitor.Warning(string.Format("Couldn't delete package directory - it's probably held open by something else: {0}", ex.Message)); - } - - } - - public void Dispose() - { - Cleanup(); - } - - public override string ToString() => DirInfo.FullName.ToString(); - } - - /// - /// The NuGet package repository. - /// - class NugetPackageRepository - { - // A list of package directories, in the order they should be searched. - private readonly string[] packageDirs; - - public NugetPackageRepository(params string[] dirs) - { - packageDirs = dirs; - } - - /// - /// Constructs a NuGet package repository, using the default locations. - /// For example, - /// $HOME/.nuget/packages, /usr/share/dotnet/sdk/NuGetFallbackFolder - /// - public NugetPackageRepository(string sourceDir) - { - var homeFolder = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); - var nugetPackages = Path.Combine(homeFolder, ".nuget", "packages"); - var nugetFallbackFolder = Environment.OSVersion.Platform == PlatformID.Win32NT ? - @"C:\Program Files\dotnet\sdk\NuGetFallbackFolder" : - "/usr/share/dotnet/sdk/NuGetFallbackFolder"; - - packageDirs = new string[] { nugetPackages, nugetFallbackFolder }; - } - - /// - /// Enumerate all available packages. - /// - public IEnumerable Packages - { - get - { - foreach (var d in packageDirs) - foreach (var p in Directory.GetDirectories(d)) - { - var name = Path.GetFileName(p); - if (!name.StartsWith('.')) - yield return new Package(p); - } - } - } - - /// - /// Tries to find a PackageFramework directory for a given package reference. - /// - /// The package reference to search for. - /// The package that was found. - /// True if a package/version/framework was found. - public bool TryFindLibs(PackageReference reference, out PackageFramework package, out ResolutionFailureReason reason) - { - var packages = packageDirs. - Where(d => Directory.Exists(Path.Combine(d, reference.Include.ToLowerInvariant()))). - Select(d => new Package(Path.Combine(d, reference.Include.ToLowerInvariant()))); - - if(!packages.Any()) - { - reason = ResolutionFailureReason.PackageNotFound; - package = null; - return false; - } - - var version = packages.Select(p => p.FindVersion(reference.Version)).FirstOrDefault(v => !(v is null)); - - if (version is null) - { - reason = ResolutionFailureReason.VersionNotFound; - package = null; - return false; - } - - package = version.BestFramework; - if(package is null) - { - reason = ResolutionFailureReason.LibsNotFound; - return false; - } - - reason = ResolutionFailureReason.Success; - return true; - } - } - - enum ResolutionFailureReason - { - Success, - PackageNotFound, - VersionNotFound, - LibsNotFound - } -} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs index 0fc207f2049..dade3909635 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs @@ -135,4 +135,49 @@ namespace Semmle.BuildAnalyser readonly string nugetExe; } + + sealed class TemporaryDirectory : IDisposable + { + public DirectoryInfo DirInfo { get; } + + public TemporaryDirectory(string name) + { + DirInfo = new DirectoryInfo(name); + DirInfo.Create(); + } + + /// + /// Computes a unique temp directory for the packages associated + /// with this source tree. Use a SHA1 of the directory name. + /// + /// + /// The full path of the temp directory. + public static string ComputeTempDirectory(string srcDir) + { + var bytes = Encoding.Unicode.GetBytes(srcDir); + + var sha1 = new SHA1CryptoServiceProvider(); + var sha = sha1.ComputeHash(bytes); + var sb = new StringBuilder(); + foreach (var b in sha.Take(8)) + sb.AppendFormat("{0:x2}", b); + + return Path.Combine(Path.GetTempPath(), "GitHub", "packages", sb.ToString()); + } + + public static TemporaryDirectory CreateTempDirectory(string source) => new TemporaryDirectory(ComputeTempDirectory(source)); + + public void Cleanup() + { + DirInfo.Delete(true); + } + + public void Dispose() + { + Cleanup(); + } + + public override string ToString() => DirInfo.FullName.ToString(); + } + } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs index 3ba368f6da0..75c2bf31e6d 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs @@ -3,11 +3,6 @@ using System.Collections.Generic; using System.Linq; using Semmle.BuildAnalyser; using Semmle.Util.Logging; -using System.IO; -using Microsoft.CodeAnalysis; -using Microsoft.CodeAnalysis.Text; - -// using Microsoft.Build.Locator; namespace Semmle.Extraction.CSharp.Standalone { @@ -87,11 +82,6 @@ namespace Semmle.Extraction.CSharp.Standalone public class Program { - void LoadSolutionFile(string file) - { - - } - static int Main(string[] args) { var options = Options.Create(args); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs index a004e59eb13..03815f7f710 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs @@ -1,4 +1,5 @@ using Semmle.Util.Logging; +using System; namespace Semmle.BuildAnalyser { @@ -9,19 +10,15 @@ namespace Semmle.BuildAnalyser { void FindingFiles(string dir); void UnresolvedReference(string id, string project); - void AnalysingProjectFiles(int count); void AnalysingSolution(string filename); void FailedProjectFile(string filename, string reason); void FailedNugetCommand(string exe, string args, string message); void NugetInstall(string package); void ResolvedReference(string filename); - void Summary(int existingSources, int usedSources, int missingSources, int references, int unresolvedReferences, int resolvedConflicts, int totalProjects, int failedProjects); + void Summary(int existingSources, int usedSources, int missingSources, int references, int unresolvedReferences, int resolvedConflicts, int totalProjects, int failedProjects, TimeSpan analysisTime); void Warning(string message); void ResolvedConflict(string asm1, string asm2); void MissingProject(string projectFile); - void Restored(string line); - void MissingPackage(string package, string version, ResolutionFailureReason reason); - void FoundPackage(string package, string version, string directory); void CommandFailed(string exe, string arguments, int exitCode); void MissingNuGet(); } @@ -52,11 +49,6 @@ namespace Semmle.BuildAnalyser logger.Log(Severity.Debug, "Unresolved {0} referenced by {1}", id, project); } - public void AnalysingProjectFiles(int count) - { - logger.Log(Severity.Info, "Analyzing project files..."); - } - public void AnalysingSolution(string filename) { logger.Log(Severity.Info, $"Analysing {filename}..."); @@ -85,7 +77,8 @@ namespace Semmle.BuildAnalyser public void Summary(int existingSources, int usedSources, int missingSources, int references, int unresolvedReferences, - int resolvedConflicts, int totalProjects, int failedProjects) + int resolvedConflicts, int totalProjects, int failedProjects, + TimeSpan analysisTime) { logger.Log(Severity.Info, ""); logger.Log(Severity.Info, "Build analysis summary:"); @@ -97,23 +90,7 @@ namespace Semmle.BuildAnalyser logger.Log(Severity.Info, "{0, 6} resolved assembly conflicts", resolvedConflicts); logger.Log(Severity.Info, "{0, 6} projects", totalProjects); logger.Log(Severity.Info, "{0, 6} missing/failed projects", failedProjects); - } - - public void Restored(string line) - { - logger.Log(Severity.Debug, $" {line}"); - } - - private static string[] reasonText = { "success", "package was not found", "the version was not found", "the package does not appear to contain any libraries" }; - - public void MissingPackage(string package, string version, ResolutionFailureReason reason) - { - logger.Log(Severity.Info, $" Couldn't find package {package} {version} because {reasonText[(int)reason]}"); - } - - public void FoundPackage(string package, string version, string directory) - { - logger.Log(Severity.Debug, $" Found package {package} {version} in {directory}"); + logger.Log(Severity.Info, "Build analysis completed in {0}", analysisTime); } public void Warning(string message) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj index 1d4ab57d2d8..b8c27e49e8d 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj @@ -21,10 +21,7 @@ - - - - + diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/SolutionFile.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/SolutionFile.cs index ffaebe360fe..b4551dd8024 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/SolutionFile.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/SolutionFile.cs @@ -11,7 +11,8 @@ namespace Semmle.BuildAnalyser class SolutionFile { readonly Microsoft.Build.Construction.SolutionFile solutionFile; - public string FullPath { get; } + + private string FullPath { get; } /// /// Read the file. diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs index a0dd41aaafb..7a14bb719fc 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs @@ -23,7 +23,8 @@ namespace Semmle.Extraction.CSharp.Entities protected override void Populate(TextWriter trapFile) { - var ns = Namespace.Create(cx, (INamespaceSymbol)cx.GetModel(Node).GetSymbolInfo(Node.Name).Symbol); + var @namespace = (INamespaceSymbol) cx.GetModel(Node).GetSymbolInfo(Node.Name).Symbol; + var ns = Namespace.Create(cx, @namespace); trapFile.namespace_declarations(this, ns); trapFile.namespace_declaration_location(this, cx.Create(Node.Name.GetLocation())); diff --git a/csharp/extractor/Semmle.Extraction/Extractor.cs b/csharp/extractor/Semmle.Extraction/Extractor.cs index e1ca23f645b..13750c1aa5c 100644 --- a/csharp/extractor/Semmle.Extraction/Extractor.cs +++ b/csharp/extractor/Semmle.Extraction/Extractor.cs @@ -50,14 +50,14 @@ namespace Semmle.Extraction /// Record a new error type. /// /// The display name of the type, qualified where possible. - /// The missing type was referenced from a source file. + /// If the missing type was referenced from a source file. void MissingType(string fqn, bool fromSource); /// /// Record an unresolved `using namespace` directive. /// /// The full name of the namespace. - /// The missing namespace was referenced from a source file. + /// If the missing namespace was referenced from a source file. void MissingNamespace(string fqn, bool fromSource); /// From b94b4b7c9132d6ef0a01a66f04a3c94daec2482b Mon Sep 17 00:00:00 2001 From: Calum Grant Date: Thu, 26 Mar 2020 17:57:14 +0000 Subject: [PATCH 3/5] C#: Fix tests --- .../BuildAnalysis.cs | 26 +++++++++---------- .../CsProjFile.cs | 3 --- .../DotNet.cs | 2 +- .../NugetPackages.cs | 23 +++++++++------- .../Program.cs | 2 +- .../SymbolExtensions.cs | 1 - .../src/semmle/code/csharp/exprs/Access.qll | 7 ++++- .../standalone/controlflow/cfg.expected | 17 ++++++++---- .../errorrecovery/ErrorCalls.expected | 2 +- .../standalone/regressions/ConstCase.expected | 5 ++-- .../standalone/regressions/ConstCase.ql | 4 +-- 11 files changed, 51 insertions(+), 41 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs index 5ec150e5af3..cdbc0315726 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs @@ -100,20 +100,20 @@ namespace Semmle.BuildAnalyser { // These files can sometimes prevent `dotnet restore` from working correctly. - using var renamer1 = new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories)); - using var renamer2 = new FileRenamer(sourceDir.GetFiles("Directory.Build.props", SearchOption.AllDirectories)); + using (new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories))) + using (new FileRenamer(sourceDir.GetFiles("Directory.Build.props", SearchOption.AllDirectories))) + { + var solutions = options.SolutionFile != null ? + new[] { options.SolutionFile } : + sourceDir.GetFiles("*.sln", SearchOption.AllDirectories).Select(d => d.FullName); - var solutions = options.SolutionFile != null ? - new[] { options.SolutionFile } : - sourceDir.GetFiles("*.sln", SearchOption.AllDirectories).Select(d => d.FullName); + RestoreSolutions(solutions); + dllDirNames.Add(PackageDirectory.DirInfo.FullName); + assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress); + AnalyseSolutions(solutions); - - RestoreSolutions(solutions); - dllDirNames.Add(PackageDirectory.DirInfo.FullName); - assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress); - AnalyseSolutions(solutions); - - usedReferences = new HashSet(assemblyCache.AllAssemblies.Select(a => a.Filename)); + usedReferences = new HashSet(assemblyCache.AllAssemblies.Select(a => a.Filename)); + } } ResolveConflicts(); @@ -254,7 +254,7 @@ namespace Semmle.BuildAnalyser unresolvedReferences[id] = projectFile; } - TemporaryDirectory PackageDirectory; + readonly TemporaryDirectory PackageDirectory; /// /// Reads all the source files and references from the given list of projects. diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs index 7394eddb39c..fc7972b6169 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs @@ -90,9 +90,6 @@ namespace Semmle.BuildAnalyser if(netCoreProjectFile) { - var frameworksNode = root.SelectNodes("/Project/PropertyGroup/TargetFrameworks").NodeList().Concat( - root.SelectNodes("/Project/PropertyGroup/TargetFramework").NodeList()).Select(node => node.InnerText); - var relativeCsIncludes2 = root.SelectNodes("/Project/ItemGroup/Compile/@Include", mgr). NodeList(). diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs index 4aad28068b0..299770a47e1 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs @@ -24,7 +24,7 @@ namespace Semmle.BuildAnalyser /// sealed class FileRenamer : IDisposable { - string[] files; + readonly string[] files; const string suffix = ".codeqlhidden"; public FileRenamer(IEnumerable oldFiles) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs index dade3909635..ddb744ab4f7 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs @@ -114,16 +114,15 @@ namespace Semmle.BuildAnalyser try { - using (var p = Process.Start(pi)) - { - string output = p.StandardOutput.ReadToEnd(); - string error = p.StandardError.ReadToEnd(); + using var p = Process.Start(pi); - p.WaitForExit(); - if (p.ExitCode != 0) - { - pm.FailedNugetCommand(pi.FileName, pi.Arguments, output + error); - } + string output = p.StandardOutput.ReadToEnd(); + string error = p.StandardError.ReadToEnd(); + + p.WaitForExit(); + if (p.ExitCode != 0) + { + pm.FailedNugetCommand(pi.FileName, pi.Arguments, output + error); } } catch (Exception ex) @@ -136,6 +135,10 @@ namespace Semmle.BuildAnalyser readonly string nugetExe; } + /// + /// A temporary directory that is created within the system temp directory. + /// When this object is disposed, the directory is deleted. + /// sealed class TemporaryDirectory : IDisposable { public DirectoryInfo DirInfo { get; } @@ -156,7 +159,7 @@ namespace Semmle.BuildAnalyser { var bytes = Encoding.Unicode.GetBytes(srcDir); - var sha1 = new SHA1CryptoServiceProvider(); + using var sha1 = new SHA1CryptoServiceProvider(); var sha = sha1.ComputeHash(bytes); var sb = new StringBuilder(); foreach (var b in sha.Take(8)) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs index 75c2bf31e6d..2dc6e2cd287 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs @@ -85,7 +85,7 @@ namespace Semmle.Extraction.CSharp.Standalone static int Main(string[] args) { var options = Options.Create(args); - options.CIL = true; + // options.CIL = true; // To do: Enable this var output = new ConsoleLogger(options.Verbosity); var a = new Analysis(output); diff --git a/csharp/extractor/Semmle.Extraction.CSharp/SymbolExtensions.cs b/csharp/extractor/Semmle.Extraction.CSharp/SymbolExtensions.cs index c2cb6a367eb..6050ad910a5 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/SymbolExtensions.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/SymbolExtensions.cs @@ -214,7 +214,6 @@ namespace Semmle.Extraction.CSharp static void BuildNamedTypeId(this INamedTypeSymbol named, Context cx, TextWriter trapFile, Action subTermAction) { bool prefixAssembly = true; - if (cx.Extractor.Standalone) prefixAssembly = false; if (named.ContainingAssembly is null) prefixAssembly = false; if (named.IsTupleType) diff --git a/csharp/ql/src/semmle/code/csharp/exprs/Access.qll b/csharp/ql/src/semmle/code/csharp/exprs/Access.qll index c0d75bfd9fe..d1e1017e0d1 100644 --- a/csharp/ql/src/semmle/code/csharp/exprs/Access.qll +++ b/csharp/ql/src/semmle/code/csharp/exprs/Access.qll @@ -388,7 +388,12 @@ library class PropertyAccessExpr extends Expr, @property_access_expr { /** Gets the target of this property access. */ Property getProperty() { expr_access(this, result) } - override string toString() { result = "access to property " + this.getProperty().getName() } + override string toString() { + result = "access to property " + this.getProperty().getName() + or + not exists(this.getProperty()) and + result = "access to property (unknown)" + } } /** diff --git a/csharp/ql/test/library-tests/standalone/controlflow/cfg.expected b/csharp/ql/test/library-tests/standalone/controlflow/cfg.expected index 06473196df9..a9eb533c152 100644 --- a/csharp/ql/test/library-tests/standalone/controlflow/cfg.expected +++ b/csharp/ql/test/library-tests/standalone/controlflow/cfg.expected @@ -7,9 +7,14 @@ | ControlFlow.cs:10:9:10:43 | Call (unknown target) | ControlFlow.cs:12:9:12:87 | ...; | | ControlFlow.cs:10:9:10:43 | call to method | ControlFlow.cs:12:9:12:87 | ...; | | ControlFlow.cs:10:9:10:44 | ...; | ControlFlow.cs:10:9:10:13 | Expression | -| ControlFlow.cs:10:22:10:22 | access to local variable v | ControlFlow.cs:10:22:10:24 | Expression | -| ControlFlow.cs:10:22:10:24 | Expression | ControlFlow.cs:10:22:10:26 | Expression | -| ControlFlow.cs:10:22:10:26 | Expression | ControlFlow.cs:10:29:10:42 | "This is true" | +| ControlFlow.cs:10:22:10:22 | access to local variable v | ControlFlow.cs:10:22:10:24 | Call (unknown target) | +| ControlFlow.cs:10:22:10:22 | access to local variable v | ControlFlow.cs:10:22:10:24 | access to property (unknown) | +| ControlFlow.cs:10:22:10:24 | Call (unknown target) | ControlFlow.cs:10:22:10:26 | Call (unknown target) | +| ControlFlow.cs:10:22:10:24 | Call (unknown target) | ControlFlow.cs:10:22:10:26 | access to property (unknown) | +| ControlFlow.cs:10:22:10:24 | access to property (unknown) | ControlFlow.cs:10:22:10:26 | Call (unknown target) | +| ControlFlow.cs:10:22:10:24 | access to property (unknown) | ControlFlow.cs:10:22:10:26 | access to property (unknown) | +| ControlFlow.cs:10:22:10:26 | Call (unknown target) | ControlFlow.cs:10:29:10:42 | "This is true" | +| ControlFlow.cs:10:22:10:26 | access to property (unknown) | ControlFlow.cs:10:29:10:42 | "This is true" | | ControlFlow.cs:10:29:10:42 | "This is true" | ControlFlow.cs:10:9:10:43 | Call (unknown target) | | ControlFlow.cs:10:29:10:42 | "This is true" | ControlFlow.cs:10:9:10:43 | call to method | | ControlFlow.cs:12:9:12:86 | Call (unknown target) | ControlFlow.cs:12:37:12:47 | Expression | @@ -20,5 +25,7 @@ | ControlFlow.cs:12:51:12:62 | access to field Empty | ControlFlow.cs:12:37:12:62 | ... = ... | | ControlFlow.cs:12:65:12:75 | Expression | ControlFlow.cs:12:79:12:79 | access to local variable v | | ControlFlow.cs:12:65:12:84 | ... = ... | ControlFlow.cs:12:35:12:86 | { ..., ... } | -| ControlFlow.cs:12:79:12:79 | access to local variable v | ControlFlow.cs:12:79:12:84 | Expression | -| ControlFlow.cs:12:79:12:84 | Expression | ControlFlow.cs:12:65:12:84 | ... = ... | +| ControlFlow.cs:12:79:12:79 | access to local variable v | ControlFlow.cs:12:79:12:84 | Call (unknown target) | +| ControlFlow.cs:12:79:12:79 | access to local variable v | ControlFlow.cs:12:79:12:84 | access to property (unknown) | +| ControlFlow.cs:12:79:12:84 | Call (unknown target) | ControlFlow.cs:12:65:12:84 | ... = ... | +| ControlFlow.cs:12:79:12:84 | access to property (unknown) | ControlFlow.cs:12:65:12:84 | ... = ... | diff --git a/csharp/ql/test/library-tests/standalone/errorrecovery/ErrorCalls.expected b/csharp/ql/test/library-tests/standalone/errorrecovery/ErrorCalls.expected index 05ae24318c3..d57c3c86bbd 100644 --- a/csharp/ql/test/library-tests/standalone/errorrecovery/ErrorCalls.expected +++ b/csharp/ql/test/library-tests/standalone/errorrecovery/ErrorCalls.expected @@ -2,5 +2,5 @@ | errors.cs:43:21:43:28 | errors.cs:43:21:43:28 | object creation of type C1 | C1 | | errors.cs:44:13:44:19 | errors.cs:44:13:44:19 | call to method m1 | m1 | | errors.cs:45:13:45:19 | errors.cs:45:13:45:19 | call to method m2 | m2 | -| errors.cs:46:13:46:38 | errors.cs:46:13:46:38 | call to method | none | +| errors.cs:46:13:46:38 | errors.cs:46:13:46:38 | call to method WriteLine | WriteLine | | errors.cs:53:17:53:25 | errors.cs:53:17:53:25 | object creation of type C2 | none | diff --git a/csharp/ql/test/library-tests/standalone/regressions/ConstCase.expected b/csharp/ql/test/library-tests/standalone/regressions/ConstCase.expected index 47045b907f6..2c8616d347a 100644 --- a/csharp/ql/test/library-tests/standalone/regressions/ConstCase.expected +++ b/csharp/ql/test/library-tests/standalone/regressions/ConstCase.expected @@ -1,2 +1,3 @@ -| regressions.cs:16:13:16:37 | case ...: | regressions.cs:16:18:16:36 | Expression | -| regressions.cs:18:13:18:37 | case ...: | regressions.cs:18:18:18:36 | Expression | +| regressions.cs:16:13:16:37 | case ...: | regressions.cs:16:18:16:36 | access to property (unknown) | +| regressions.cs:18:13:18:37 | case ...: | regressions.cs:18:18:18:36 | access to property (unknown) | +| regressions.cs:20:13:20:23 | case ...: | regressions.cs:20:18:20:22 | Int32 x | diff --git a/csharp/ql/test/library-tests/standalone/regressions/ConstCase.ql b/csharp/ql/test/library-tests/standalone/regressions/ConstCase.ql index 7ee7754e91e..6c5d54518fe 100644 --- a/csharp/ql/test/library-tests/standalone/regressions/ConstCase.ql +++ b/csharp/ql/test/library-tests/standalone/regressions/ConstCase.ql @@ -1,7 +1,5 @@ import csharp from Case c, Expr e -where - e = c.getPattern().stripCasts() and - (e instanceof @unknown_expr or e instanceof ConstantPatternExpr) +where e = c.getPattern().stripCasts() select c, e From 9481fada510f714397d9fe9c7019ddaae3cd9a47 Mon Sep 17 00:00:00 2001 From: Calum Grant Date: Thu, 2 Apr 2020 20:29:45 +0100 Subject: [PATCH 4/5] C#: Address review comments. --- .../BuildAnalysis.cs | 112 ++++++++++-------- .../CsProjFile.cs | 55 ++++----- .../DotNet.cs | 29 +---- .../NugetPackages.cs | 54 +-------- .../Program.cs | 12 +- .../ProgressMonitor.cs | 2 +- ...Semmle.Extraction.CSharp.Standalone.csproj | 1 + 7 files changed, 94 insertions(+), 171 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs index cdbc0315726..2894222ca89 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs @@ -1,9 +1,13 @@ -using System; +using Semmle.Util; +using System; using System.Collections.Generic; using System.IO; using System.Linq; using Semmle.Extraction.CSharp.Standalone; using System.Threading.Tasks; +using System.Collections.Concurrent; +using System.Text; +using System.Security.Cryptography; namespace Semmle.BuildAnalyser { @@ -42,20 +46,18 @@ namespace Semmle.BuildAnalyser /// /// Main implementation of the build analysis. /// - class BuildAnalysis : IBuildAnalysis + class BuildAnalysis : IBuildAnalysis, IDisposable { private readonly AssemblyCache assemblyCache; private readonly NugetPackages nuget; private readonly IProgressMonitor progressMonitor; - private HashSet usedReferences = new HashSet(); - private readonly HashSet usedSources = new HashSet(); - private readonly HashSet missingSources = new HashSet(); - private readonly Dictionary unresolvedReferences = new Dictionary(); + private readonly IDictionary usedReferences = new ConcurrentDictionary(); + private readonly IDictionary sources = new ConcurrentDictionary(); + private readonly IDictionary unresolvedReferences = new ConcurrentDictionary(); private readonly DirectoryInfo sourceDir; private int failedProjects, succeededProjects; private readonly string[] allSources; private int conflictedReferences = 0; - private readonly object mutex = new object(); /// /// Performs a C# build analysis. @@ -77,7 +79,7 @@ namespace Semmle.BuildAnalyser ToArray(); var dllDirNames = options.DllDirs.Select(Path.GetFullPath).ToList(); - PackageDirectory = TemporaryDirectory.CreateTempDirectory(sourceDir.FullName); + PackageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName)); if (options.UseNuGet) { @@ -97,23 +99,22 @@ namespace Semmle.BuildAnalyser { dllDirNames.Add(Runtime.Runtimes.First()); } - + + // These files can sometimes prevent `dotnet restore` from working correctly. + using (new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories))) + using (new FileRenamer(sourceDir.GetFiles("Directory.Build.props", SearchOption.AllDirectories))) { - // These files can sometimes prevent `dotnet restore` from working correctly. - using (new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories))) - using (new FileRenamer(sourceDir.GetFiles("Directory.Build.props", SearchOption.AllDirectories))) - { - var solutions = options.SolutionFile != null ? - new[] { options.SolutionFile } : - sourceDir.GetFiles("*.sln", SearchOption.AllDirectories).Select(d => d.FullName); + var solutions = options.SolutionFile != null ? + new[] { options.SolutionFile } : + sourceDir.GetFiles("*.sln", SearchOption.AllDirectories).Select(d => d.FullName); - RestoreSolutions(solutions); - dllDirNames.Add(PackageDirectory.DirInfo.FullName); - assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress); - AnalyseSolutions(solutions); + RestoreSolutions(solutions); + dllDirNames.Add(PackageDirectory.DirInfo.FullName); + assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress); + AnalyseSolutions(solutions); - usedReferences = new HashSet(assemblyCache.AllAssemblies.Select(a => a.Filename)); - } + foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename)) + UseReference(filename); } ResolveConflicts(); @@ -124,7 +125,7 @@ namespace Semmle.BuildAnalyser } // Output the findings - foreach (var r in usedReferences) + foreach (var r in usedReferences.Keys) { progressMonitor.ResolvedReference(r); } @@ -146,6 +147,25 @@ namespace Semmle.BuildAnalyser DateTime.Now - startTime); } + /// + /// Computes a unique temp directory for the packages associated + /// with this source tree. Use a SHA1 of the directory name. + /// + /// + /// The full path of the temp directory. + private static string ComputeTempDirectory(string srcDir) + { + var bytes = Encoding.Unicode.GetBytes(srcDir); + + using var sha1 = new SHA1CryptoServiceProvider(); + var sha = sha1.ComputeHash(bytes); + var sb = new StringBuilder(); + foreach (var b in sha.Take(8)) + sb.AppendFormat("{0:x2}", b); + + return Path.Combine(Path.GetTempPath(), "GitHub", "packages", sb.ToString()); + } + /// /// Resolves conflicts between all of the resolved references. /// If the same assembly name is duplicated with different versions, @@ -154,7 +174,7 @@ namespace Semmle.BuildAnalyser void ResolveConflicts() { var sortedReferences = usedReferences. - Select(r => assemblyCache.GetAssemblyInfo(r)). + Select(r => assemblyCache.GetAssemblyInfo(r.Key)). OrderBy(r => r.Version). ToArray(); @@ -165,7 +185,9 @@ namespace Semmle.BuildAnalyser finalAssemblyList[r.Name] = r; // Update the used references list - usedReferences = new HashSet(finalAssemblyList.Select(r => r.Value.Filename)); + usedReferences.Clear(); + foreach (var r in finalAssemblyList.Select(r => r.Value.Filename)) + UseReference(r); // Report the results foreach (var r in sortedReferences) @@ -194,8 +216,7 @@ namespace Semmle.BuildAnalyser /// The filename of the reference. void UseReference(string reference) { - lock (mutex) - usedReferences.Add(reference); + usedReferences[reference] = true; } /// @@ -204,27 +225,18 @@ namespace Semmle.BuildAnalyser /// The source file. void UseSource(FileInfo sourceFile) { - if (sourceFile.Exists) - { - lock(mutex) - usedSources.Add(sourceFile.FullName); - } - else - { - lock(mutex) - missingSources.Add(sourceFile.FullName); - } + sources[sourceFile.FullName] = sourceFile.Exists; } /// /// The list of resolved reference files. /// - public IEnumerable ReferenceFiles => this.usedReferences; + public IEnumerable ReferenceFiles => this.usedReferences.Keys; /// /// The list of source files used in projects. /// - public IEnumerable ProjectSourceFiles => usedSources; + public IEnumerable ProjectSourceFiles => sources.Where(s => s.Value).Select(s => s.Key); /// /// All of the source files in the source directory. @@ -240,7 +252,7 @@ namespace Semmle.BuildAnalyser /// List of source files which were mentioned in project files but /// do not exist on the file system. /// - public IEnumerable MissingSourceFiles => missingSources; + public IEnumerable MissingSourceFiles => sources.Where(s => !s.Value).Select(s => s.Key); /// /// Record that a particular reference couldn't be resolved. @@ -250,8 +262,7 @@ namespace Semmle.BuildAnalyser /// The project file making the reference. void UnresolvedReference(string id, string projectFile) { - lock(mutex) - unresolvedReferences[id] = projectFile; + unresolvedReferences[id] = projectFile; } readonly TemporaryDirectory PackageDirectory; @@ -276,7 +287,7 @@ namespace Semmle.BuildAnalyser try { - IProjectFile csProj = new CsProjFile(project); + var csProj = new CsProjFile(project); foreach (var @ref in csProj.References) { @@ -309,14 +320,6 @@ namespace Semmle.BuildAnalyser } - /// - /// Delete packages directory. - /// - public void Cleanup() - { - PackageDirectory?.Cleanup(); - } - void Restore(string projectOrSolution) { int exit = DotNet.RestoreToDirectory(projectOrSolution, PackageDirectory.DirInfo.FullName); @@ -345,7 +348,7 @@ namespace Semmle.BuildAnalyser { var sln = new SolutionFile(solutionFile); progressMonitor.AnalysingSolution(solutionFile); - AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).Where(p => p.Exists).ToArray()); + AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).Where(p => p.Exists)); } catch (Microsoft.Build.Exceptions.InvalidProjectFileException ex) { @@ -353,5 +356,10 @@ namespace Semmle.BuildAnalyser } }); } + + public void Dispose() + { + PackageDirectory?.Dispose(); + } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs index fc7972b6169..1083c9b6257 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs @@ -5,21 +5,14 @@ using System.Xml; namespace Semmle.BuildAnalyser { - interface IProjectFile - { - IEnumerable References { get; } - - IEnumerable Sources { get; } - } - /// /// Represents a .csproj file and reads information from it. /// - class CsProjFile : IProjectFile + class CsProjFile { - public string Filename { get; } + private string Filename { get; } - public string Directory => Path.GetDirectoryName(Filename); + private string Directory => Path.GetDirectoryName(Filename); /// /// Reads the .csproj file. @@ -52,7 +45,7 @@ namespace Semmle.BuildAnalyser /// and there seems to be no way to make it succeed. Fails on Linux. /// /// The file to read. - public void ReadMsBuildProject(FileInfo filename) + private void ReadMsBuildProject(FileInfo filename) { var msbuildProject = new Microsoft.Build.Execution.ProjectInstance(filename.FullName); @@ -75,7 +68,7 @@ namespace Semmle.BuildAnalyser /// fallback if ReadMsBuildProject() fails. /// /// The .csproj file. - public void ReadProjectFileAsXml(FileInfo filename) + private void ReadProjectFileAsXml(FileInfo filename) { var projFile = new XmlDocument(); var mgr = new XmlNamespaceManager(projFile.NameTable); @@ -88,15 +81,15 @@ namespace Semmle.BuildAnalyser bool netCoreProjectFile = root.GetAttribute("Sdk") == "Microsoft.NET.Sdk"; - if(netCoreProjectFile) + if (netCoreProjectFile) { - var relativeCsIncludes2 = + var relativeCsIncludes = root.SelectNodes("/Project/ItemGroup/Compile/@Include", mgr). NodeList(). Select(node => node.Value). ToArray(); - var explicitCsFiles = relativeCsIncludes2. + var explicitCsFiles = relativeCsIncludes. Select(cs => Path.DirectorySeparatorChar == '/' ? cs.Replace("\\", "/") : cs). Select(f => Path.GetFullPath(Path.Combine(projDir.FullName, f))); @@ -105,25 +98,27 @@ namespace Semmle.BuildAnalyser csFiles = explicitCsFiles.Concat(additionalCsFiles).ToArray(); references = new string[0]; - return; } + else + { - references = - root.SelectNodes("/msbuild:Project/msbuild:ItemGroup/msbuild:Reference/@Include", mgr). - NodeList(). - Select(node => node.Value). - ToArray(); + references = + root.SelectNodes("/msbuild:Project/msbuild:ItemGroup/msbuild:Reference/@Include", mgr). + NodeList(). + Select(node => node.Value). + ToArray(); - var relativeCsIncludes = - root.SelectNodes("/msbuild:Project/msbuild:ItemGroup/msbuild:Compile/@Include", mgr). - NodeList(). - Select(node => node.Value). - ToArray(); + var relativeCsIncludes = + root.SelectNodes("/msbuild:Project/msbuild:ItemGroup/msbuild:Compile/@Include", mgr). + NodeList(). + Select(node => node.Value). + ToArray(); - csFiles = relativeCsIncludes. - Select(cs => Path.DirectorySeparatorChar == '/' ? cs.Replace("\\", "/") : cs). - Select(f => Path.GetFullPath(Path.Combine(projDir.FullName, f))). - ToArray(); + csFiles = relativeCsIncludes. + Select(cs => Path.DirectorySeparatorChar == '/' ? cs.Replace("\\", "/") : cs). + Select(f => Path.GetFullPath(Path.Combine(projDir.FullName, f))). + ToArray(); + } } string[] references; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs index 299770a47e1..6edd217af8d 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs @@ -13,36 +13,9 @@ namespace Semmle.BuildAnalyser { public static int RestoreToDirectory(string projectOrSolutionFile, string packageDirectory) { - using var proc = Process.Start("dotnet", $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\""); + using var proc = Process.Start("dotnet", $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true"); proc.WaitForExit(); return proc.ExitCode; } } - - /// - /// Utility to temporarily rename a set of files. - /// - sealed class FileRenamer : IDisposable - { - readonly string[] files; - const string suffix = ".codeqlhidden"; - - public FileRenamer(IEnumerable oldFiles) - { - files = oldFiles.Select(f => f.FullName).ToArray(); - - foreach(var file in files) - { - File.Move(file, file + suffix); - } - } - - public void Dispose() - { - foreach (var file in files) - { - File.Move(file + suffix, file); - } - } - } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs index ddb744ab4f7..2ea3afb6c69 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs @@ -1,10 +1,9 @@ -using System; +using Semmle.Util; +using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; -using System.Security.Cryptography; -using System.Text; namespace Semmle.BuildAnalyser { @@ -134,53 +133,4 @@ namespace Semmle.BuildAnalyser readonly string nugetExe; } - - /// - /// A temporary directory that is created within the system temp directory. - /// When this object is disposed, the directory is deleted. - /// - sealed class TemporaryDirectory : IDisposable - { - public DirectoryInfo DirInfo { get; } - - public TemporaryDirectory(string name) - { - DirInfo = new DirectoryInfo(name); - DirInfo.Create(); - } - - /// - /// Computes a unique temp directory for the packages associated - /// with this source tree. Use a SHA1 of the directory name. - /// - /// - /// The full path of the temp directory. - public static string ComputeTempDirectory(string srcDir) - { - var bytes = Encoding.Unicode.GetBytes(srcDir); - - using var sha1 = new SHA1CryptoServiceProvider(); - var sha = sha1.ComputeHash(bytes); - var sb = new StringBuilder(); - foreach (var b in sha.Take(8)) - sb.AppendFormat("{0:x2}", b); - - return Path.Combine(Path.GetTempPath(), "GitHub", "packages", sb.ToString()); - } - - public static TemporaryDirectory CreateTempDirectory(string source) => new TemporaryDirectory(ComputeTempDirectory(source)); - - public void Cleanup() - { - DirInfo.Delete(true); - } - - public void Dispose() - { - Cleanup(); - } - - public override string ToString() => DirInfo.FullName.ToString(); - } - } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs index 2dc6e2cd287..106771faef2 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs @@ -23,7 +23,7 @@ namespace Semmle.Extraction.CSharp.Standalone /// /// Searches for source/references and creates separate extractions. /// - class Analysis + class Analysis : IDisposable { readonly ILogger logger; @@ -71,12 +71,9 @@ namespace Semmle.Extraction.CSharp.Standalone projectExtraction.Sources.AddRange(options.SolutionFile == null ? buildAnalysis.AllSourceFiles : buildAnalysis.ProjectSourceFiles); } - /// - /// Delete any Nuget assemblies. - /// - public void Cleanup() + public void Dispose() { - buildAnalysis.Cleanup(); + buildAnalysis.Dispose(); } }; @@ -87,7 +84,7 @@ namespace Semmle.Extraction.CSharp.Standalone var options = Options.Create(args); // options.CIL = true; // To do: Enable this var output = new ConsoleLogger(options.Verbosity); - var a = new Analysis(output); + using var a = new Analysis(output); if (options.Help) { @@ -123,7 +120,6 @@ namespace Semmle.Extraction.CSharp.Standalone output.Log(Severity.Info, $"Extraction completed in {DateTime.Now-start}"); } - a.Cleanup(); return 0; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs index 03815f7f710..5bbe1e3a5b2 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs @@ -51,7 +51,7 @@ namespace Semmle.BuildAnalyser public void AnalysingSolution(string filename) { - logger.Log(Severity.Info, $"Analysing {filename}..."); + logger.Log(Severity.Info, $"Analyzing {filename}..."); } public void FailedProjectFile(string filename, string reason) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj index b8c27e49e8d..f9efd0d9ebb 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj @@ -14,6 +14,7 @@ + From adde52d33cb1fa03ffb605f610ab221b83fcd85e Mon Sep 17 00:00:00 2001 From: Calum Grant Date: Fri, 3 Apr 2020 10:36:47 +0100 Subject: [PATCH 5/5] C#: Add missing files --- csharp/extractor/Semmle.Util/FileRenamer.cs | 34 +++++++++++++++++++ .../Semmle.Util/TemporaryDirectory.cs | 30 ++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 csharp/extractor/Semmle.Util/FileRenamer.cs create mode 100644 csharp/extractor/Semmle.Util/TemporaryDirectory.cs diff --git a/csharp/extractor/Semmle.Util/FileRenamer.cs b/csharp/extractor/Semmle.Util/FileRenamer.cs new file mode 100644 index 00000000000..ad5001f7e13 --- /dev/null +++ b/csharp/extractor/Semmle.Util/FileRenamer.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace Semmle.Util +{ + /// + /// Utility to temporarily rename a set of files. + /// + public sealed class FileRenamer : IDisposable + { + readonly string[] files; + const string suffix = ".codeqlhidden"; + + public FileRenamer(IEnumerable oldFiles) + { + files = oldFiles.Select(f => f.FullName).ToArray(); + + foreach (var file in files) + { + File.Move(file, file + suffix); + } + } + + public void Dispose() + { + foreach (var file in files) + { + File.Move(file + suffix, file); + } + } + } +} diff --git a/csharp/extractor/Semmle.Util/TemporaryDirectory.cs b/csharp/extractor/Semmle.Util/TemporaryDirectory.cs new file mode 100644 index 00000000000..a155372c9d8 --- /dev/null +++ b/csharp/extractor/Semmle.Util/TemporaryDirectory.cs @@ -0,0 +1,30 @@ +using System; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; + +namespace Semmle.Util +{ + /// + /// A temporary directory that is created within the system temp directory. + /// When this object is disposed, the directory is deleted. + /// + public sealed class TemporaryDirectory : IDisposable + { + public DirectoryInfo DirInfo { get; } + + public TemporaryDirectory(string name) + { + DirInfo = new DirectoryInfo(name); + DirInfo.Create(); + } + + public void Dispose() + { + DirInfo.Delete(true); + } + + public override string ToString() => DirInfo.FullName.ToString(); + } +}