Merge pull request #3136 from calumgrant/cs/buildless-extraction

C#: Improvements to buildless extraction

.gitignore

@@ -19,5 +19,6 @@
 
 # It's useful (though not required) to be able to unpack codeql in the ql checkout itself
 /codeql/
-.vscode/settings.json
+
 csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/launchSettings.json
+.vscode

csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs

@@ -163,7 +163,19 @@ namespace Semmle.BuildAnalyser
         /// </summary>
         /// <param name="filepath">The filename to query.</param>
         /// <returns>The assembly info.</returns>
-        public AssemblyInfo GetAssemblyInfo(string filepath) => assemblyInfo[filepath];
+        public AssemblyInfo GetAssemblyInfo(string filepath)
+        {
+            if(assemblyInfo.TryGetValue(filepath, out var info))
+            {
+                return info;
+            }
+            else
+            {
+                info = AssemblyInfo.ReadFromFile(filepath);
+                assemblyInfo.Add(filepath, info);
+                return info;
+            }
+        }
 
         // List of pending DLLs to index.
         readonly List<string> dlls = new List<string>();

csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs

@@ -1,10 +1,13 @@
-using System;
+using Semmle.Util;
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using System.Runtime.InteropServices;
-using Semmle.Util;
 using Semmle.Extraction.CSharp.Standalone;
+using System.Threading.Tasks;
+using System.Collections.Concurrent;
+using System.Text;
+using System.Security.Cryptography;
 
 namespace Semmle.BuildAnalyser
 {
@@ -43,19 +46,18 @@ namespace Semmle.BuildAnalyser
     /// <summary>
     /// Main implementation of the build analysis.
     /// </summary>
-    class BuildAnalysis : IBuildAnalysis
+    class BuildAnalysis : IBuildAnalysis, IDisposable
     {
-        readonly AssemblyCache assemblyCache;
+        private readonly AssemblyCache assemblyCache;
-        readonly NugetPackages nuget;
+        private readonly NugetPackages nuget;
-        readonly IProgressMonitor progressMonitor;
+        private readonly IProgressMonitor progressMonitor;
-        HashSet<string> usedReferences = new HashSet<string>();
+        private readonly IDictionary<string, bool> usedReferences = new ConcurrentDictionary<string, bool>();
-        readonly HashSet<string> usedSources = new HashSet<string>();
+        private readonly IDictionary<string, bool> sources = new ConcurrentDictionary<string, bool>();
-        readonly HashSet<string> missingSources = new HashSet<string>();
+        private readonly IDictionary<string, string> unresolvedReferences = new ConcurrentDictionary<string, string>();
-        readonly Dictionary<string, string> unresolvedReferences = new Dictionary<string, string>();
+        private readonly DirectoryInfo sourceDir;
-        readonly DirectoryInfo sourceDir;
+        private int failedProjects, succeededProjects;
-        int failedProjects, succeededProjects;
+        private readonly string[] allSources;
-        readonly string[] allSources;
+        private int conflictedReferences = 0;
-        int conflictedReferences = 0;
 
         /// <summary>
         /// Performs a C# build analysis.
@@ -64,6 +66,8 @@ namespace Semmle.BuildAnalyser
         /// <param name="progress">Display of analysis progress.</param>
         public BuildAnalysis(Options options, IProgressMonitor progress)
         {
+            var startTime = DateTime.Now;
+
             progressMonitor = progress;
             sourceDir = new DirectoryInfo(options.SrcDir);
 
@@ -74,36 +78,43 @@ namespace Semmle.BuildAnalyser
                 Where(d => !options.ExcludesFile(d)).
                 ToArray();
 
-            var dllDirNames = options.DllDirs.Select(Path.GetFullPath);
+            var dllDirNames = options.DllDirs.Select(Path.GetFullPath).ToList();
+            PackageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
 
             if (options.UseNuGet)
             {
-                nuget = new NugetPackages(sourceDir.FullName);
+                try
-                ReadNugetFiles();
+                {
-                dllDirNames = dllDirNames.Concat(Enumerators.Singleton(nuget.PackageDirectory));
+                    nuget = new NugetPackages(sourceDir.FullName, PackageDirectory);
+                    ReadNugetFiles();
+                }
+                catch(FileNotFoundException)
+                {
+                    progressMonitor.MissingNuGet();
+                }
             }
 
             // Find DLLs in the .Net Framework
             if (options.ScanNetFrameworkDlls)
             {
-                dllDirNames = dllDirNames.Concat(Runtime.Runtimes.Take(1));
+                dllDirNames.Add(Runtime.Runtimes.First());
             }
 
-            assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress);
+            // These files can sometimes prevent `dotnet restore` from working correctly.
+            using (new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories)))
+            using (new FileRenamer(sourceDir.GetFiles("Directory.Build.props", SearchOption.AllDirectories)))
+            {
+                var solutions = options.SolutionFile != null ?
+                        new[] { options.SolutionFile } :
+                        sourceDir.GetFiles("*.sln", SearchOption.AllDirectories).Select(d => d.FullName);
 
-            // Analyse all .csproj files in the source tree.
+                RestoreSolutions(solutions);
-            if (options.SolutionFile != null)
+                dllDirNames.Add(PackageDirectory.DirInfo.FullName);
-            {
+                assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress);
-                AnalyseSolution(options.SolutionFile);
+                AnalyseSolutions(solutions);
-            }
-            else if (options.AnalyseCsProjFiles)
-            {
-                AnalyseProjectFiles();
-            }
 
-            if (!options.AnalyseCsProjFiles)
+                foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename))
-            {
+                    UseReference(filename);
-                usedReferences = new HashSet<string>(assemblyCache.AllAssemblies.Select(a => a.Filename));
             }
 
             ResolveConflicts();
@@ -114,7 +125,7 @@ namespace Semmle.BuildAnalyser
             }
 
             // Output the findings
-            foreach (var r in usedReferences)
+            foreach (var r in usedReferences.Keys)
             {
                 progressMonitor.ResolvedReference(r);
             }
@@ -132,7 +143,27 @@ namespace Semmle.BuildAnalyser
                 UnresolvedReferences.Count(),
                 conflictedReferences,
                 succeededProjects + failedProjects,
-                failedProjects);
+                failedProjects,
+                DateTime.Now - startTime);
+        }
+
+        /// <summary>
+        /// Computes a unique temp directory for the packages associated
+        /// with this source tree. Use a SHA1 of the directory name.
+        /// </summary>
+        /// <param name="srcDir"></param>
+        /// <returns>The full path of the temp directory.</returns>
+        private static string ComputeTempDirectory(string srcDir)
+        {
+            var bytes = Encoding.Unicode.GetBytes(srcDir);
+
+            using var sha1 = new SHA1CryptoServiceProvider();
+            var sha = sha1.ComputeHash(bytes);
+            var sb = new StringBuilder();
+            foreach (var b in sha.Take(8))
+                sb.AppendFormat("{0:x2}", b);
+
+            return Path.Combine(Path.GetTempPath(), "GitHub", "packages", sb.ToString());
         }
 
         /// <summary>
@@ -143,7 +174,7 @@ namespace Semmle.BuildAnalyser
         void ResolveConflicts()
         {
             var sortedReferences = usedReferences.
-                Select(r => assemblyCache.GetAssemblyInfo(r)).
+                Select(r => assemblyCache.GetAssemblyInfo(r.Key)).
                 OrderBy(r => r.Version).
                 ToArray();
 
@@ -154,7 +185,9 @@ namespace Semmle.BuildAnalyser
                 finalAssemblyList[r.Name] = r;
 
             // Update the used references list
-            usedReferences = new HashSet<string>(finalAssemblyList.Select(r => r.Value.Filename));
+            usedReferences.Clear();
+            foreach (var r in finalAssemblyList.Select(r => r.Value.Filename))
+                UseReference(r);
 
             // Report the results
             foreach (var r in sortedReferences)
@@ -183,7 +216,7 @@ namespace Semmle.BuildAnalyser
         /// <param name="reference">The filename of the reference.</param>
         void UseReference(string reference)
         {
-            usedReferences.Add(reference);
+            usedReferences[reference] = true;
         }
 
         /// <summary>
@@ -192,25 +225,18 @@ namespace Semmle.BuildAnalyser
         /// <param name="sourceFile">The source file.</param>
         void UseSource(FileInfo sourceFile)
         {
-            if (sourceFile.Exists)
+            sources[sourceFile.FullName] = sourceFile.Exists;
-            {
-                usedSources.Add(sourceFile.FullName);
-            }
-            else
-            {
-                missingSources.Add(sourceFile.FullName);
-            }
         }
 
         /// <summary>
         /// The list of resolved reference files.
         /// </summary>
-        public IEnumerable<string> ReferenceFiles => this.usedReferences;
+        public IEnumerable<string> ReferenceFiles => this.usedReferences.Keys;
 
         /// <summary>
         /// The list of source files used in projects.
         /// </summary>
-        public IEnumerable<string> ProjectSourceFiles => usedSources;
+        public IEnumerable<string> ProjectSourceFiles => sources.Where(s => s.Value).Select(s => s.Key);
 
         /// <summary>
         /// All of the source files in the source directory.
@@ -226,7 +252,7 @@ namespace Semmle.BuildAnalyser
         /// List of source files which were mentioned in project files but
         /// do not exist on the file system.
         /// </summary>
-        public IEnumerable<string> MissingSourceFiles => missingSources;
+        public IEnumerable<string> MissingSourceFiles => sources.Where(s => !s.Value).Select(s => s.Key);
 
         /// <summary>
         /// Record that a particular reference couldn't be resolved.
@@ -239,74 +265,101 @@ namespace Semmle.BuildAnalyser
             unresolvedReferences[id] = projectFile;
         }
 
-        /// <summary>
+        readonly TemporaryDirectory PackageDirectory;
-        /// Performs an analysis of all .csproj files.
-        /// </summary>
-        void AnalyseProjectFiles()
-        {
-            AnalyseProjectFiles(sourceDir.GetFiles("*.csproj", SearchOption.AllDirectories));
-        }
 
         /// <summary>
         /// Reads all the source files and references from the given list of projects.
         /// </summary>
         /// <param name="projectFiles">The list of projects to analyse.</param>
-        void AnalyseProjectFiles(FileInfo[] projectFiles)
+        void AnalyseProjectFiles(IEnumerable<FileInfo> projectFiles)
         {
-            progressMonitor.AnalysingProjectFiles(projectFiles.Count());
-
             foreach (var proj in projectFiles)
+                AnalyseProject(proj);
+        }
+
+        void AnalyseProject(FileInfo project)
+        {
+            if(!project.Exists)
             {
-                try
+                progressMonitor.MissingProject(project.FullName);
-                {
+                return;
-                    var csProj = new CsProjFile(proj);
+            }
 
-                    foreach (var @ref in csProj.References)
+            try
-                    {
+            {
-                        AssemblyInfo resolved = assemblyCache.ResolveReference(@ref);
+                var csProj = new CsProjFile(project);
-                        if (!resolved.Valid)
-                        {
-                            UnresolvedReference(@ref, proj.FullName);
-                        }
-                        else
-                        {
-                            UseReference(resolved.Filename);
-                        }
-                    }
 
-                    foreach (var src in csProj.Sources)
+                foreach (var @ref in csProj.References)
-                    {
-                        // Make a note of which source files the projects use.
-                        // This information doesn't affect the build but is dumped
-                        // as diagnostic output.
-                        UseSource(new FileInfo(src));
-                    }
-                    ++succeededProjects;
-                }
-                catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
                 {
-                    ++failedProjects;
+                    AssemblyInfo resolved = assemblyCache.ResolveReference(@ref);
-                    progressMonitor.FailedProjectFile(proj.FullName, ex.Message);
+                    if (!resolved.Valid)
+                    {
+                        UnresolvedReference(@ref, project.FullName);
+                    }
+                    else
+                    {
+                        UseReference(resolved.Filename);
+                    }
                 }
+
+                foreach (var src in csProj.Sources)
+                {
+                    // Make a note of which source files the projects use.
+                    // This information doesn't affect the build but is dumped
+                    // as diagnostic output.
+                    UseSource(new FileInfo(src));
+                }
+
+                ++succeededProjects;
+            }
+            catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
+            {
+                ++failedProjects;
+                progressMonitor.FailedProjectFile(project.FullName, ex.Message);
+            }
+
+        }
+
+        void Restore(string projectOrSolution)
+        {
+            int exit = DotNet.RestoreToDirectory(projectOrSolution, PackageDirectory.DirInfo.FullName);
+            switch(exit)
+            {
+                case 0:
+                case 1:
+                    // No errors
+                    break;
+                default:
+                    progressMonitor.CommandFailed("dotnet", $"restore \"{projectOrSolution}\"", exit);
+                    break;
             }
         }
 
-        /// <summary>
+        public void RestoreSolutions(IEnumerable<string> solutions)
-        /// Delete packages directory.
-        /// </summary>
-        public void Cleanup()
         {
-            if (nuget != null) nuget.Cleanup(progressMonitor);
+            Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = 4 }, Restore);
         }
 
-        /// <summary>
+        public void AnalyseSolutions(IEnumerable<string> solutions)
-        /// Analyse all project files in a given solution only.
-        /// </summary>
-        /// <param name="solutionFile">The filename of the solution.</param>
-        public void AnalyseSolution(string solutionFile)
         {
-            var sln = new SolutionFile(solutionFile);
+            Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = 4 } , solutionFile =>
-            AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).ToArray());
+            {
+                try
+                {
+                    var sln = new SolutionFile(solutionFile);
+                    progressMonitor.AnalysingSolution(solutionFile);
+                    AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).Where(p => p.Exists));
+                }
+                catch (Microsoft.Build.Exceptions.InvalidProjectFileException ex)
+                {
+                    progressMonitor.FailedProjectFile(solutionFile, ex.BaseMessage);
+                }
+            });
+        }
+
+        public void Dispose()
+        {
+            PackageDirectory?.Dispose();
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs

@@ -10,12 +10,18 @@ namespace Semmle.BuildAnalyser
     /// </summary>
     class CsProjFile
     {
+        private string Filename { get; }
+
+        private string Directory => Path.GetDirectoryName(Filename);
+
         /// <summary>
         /// Reads the .csproj file.
         /// </summary>
         /// <param name="filename">The .csproj file.</param>
         public CsProjFile(FileInfo filename)
         {
+            Filename = filename.FullName;
+
             try
             {
                 // This can fail if the .csproj is invalid or has
@@ -39,7 +45,7 @@ namespace Semmle.BuildAnalyser
         /// and there seems to be no way to make it succeed. Fails on Linux.
         /// </summary>
         /// <param name="filename">The file to read.</param>
-        public void ReadMsBuildProject(FileInfo filename)
+        private void ReadMsBuildProject(FileInfo filename)
         {
             var msbuildProject = new Microsoft.Build.Execution.ProjectInstance(filename.FullName);
 
@@ -62,7 +68,7 @@ namespace Semmle.BuildAnalyser
         /// fallback if ReadMsBuildProject() fails.
         /// </summary>
         /// <param name="filename">The .csproj file.</param>
-        public void ReadProjectFileAsXml(FileInfo filename)
+        private void ReadProjectFileAsXml(FileInfo filename)
         {
             var projFile = new XmlDocument();
             var mgr = new XmlNamespaceManager(projFile.NameTable);
@@ -71,22 +77,48 @@ namespace Semmle.BuildAnalyser
             var projDir = filename.Directory;
             var root = projFile.DocumentElement;
 
-            references =
+            // Figure out if it's dotnet core
-                root.SelectNodes("/msbuild:Project/msbuild:ItemGroup/msbuild:Reference/@Include", mgr).
-                NodeList().
-                Select(node => node.Value).
-                ToArray();
 
-            var relativeCsIncludes =
+            bool netCoreProjectFile = root.GetAttribute("Sdk") == "Microsoft.NET.Sdk";
-                root.SelectNodes("/msbuild:Project/msbuild:ItemGroup/msbuild:Compile/@Include", mgr).
-                NodeList().
-                Select(node => node.Value).
-                ToArray();
 
-            csFiles = relativeCsIncludes.
+            if (netCoreProjectFile)
-                Select(cs => Path.DirectorySeparatorChar == '/' ? cs.Replace("\\", "/") : cs).
+            {
-                Select(f => Path.GetFullPath(Path.Combine(projDir.FullName, f))).
+                var relativeCsIncludes =
-                ToArray();
+                    root.SelectNodes("/Project/ItemGroup/Compile/@Include", mgr).
+                    NodeList().
+                    Select(node => node.Value).
+                    ToArray();
+
+                var explicitCsFiles = relativeCsIncludes.
+                    Select(cs => Path.DirectorySeparatorChar == '/' ? cs.Replace("\\", "/") : cs).
+                    Select(f => Path.GetFullPath(Path.Combine(projDir.FullName, f)));
+
+                var additionalCsFiles = System.IO.Directory.GetFiles(Directory, "*.cs", SearchOption.AllDirectories);
+
+                csFiles = explicitCsFiles.Concat(additionalCsFiles).ToArray();
+
+                references = new string[0];
+            }
+            else
+            {
+
+                references =
+                    root.SelectNodes("/msbuild:Project/msbuild:ItemGroup/msbuild:Reference/@Include", mgr).
+                    NodeList().
+                    Select(node => node.Value).
+                    ToArray();
+
+                var relativeCsIncludes =
+                    root.SelectNodes("/msbuild:Project/msbuild:ItemGroup/msbuild:Compile/@Include", mgr).
+                    NodeList().
+                    Select(node => node.Value).
+                    ToArray();
+
+                csFiles = relativeCsIncludes.
+                    Select(cs => Path.DirectorySeparatorChar == '/' ? cs.Replace("\\", "/") : cs).
+                    Select(f => Path.GetFullPath(Path.Combine(projDir.FullName, f))).
+                    ToArray();
+            }
         }
 
         string[] references;

csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+
+namespace Semmle.BuildAnalyser
+{
+    /// <summary>
+    /// Utilities to run the "dotnet" command.
+    /// </summary>
+    static class DotNet
+    {
+        public static int RestoreToDirectory(string projectOrSolutionFile, string packageDirectory)
+        {
+            using var proc = Process.Start("dotnet", $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true");
+            proc.WaitForExit();
+            return proc.ExitCode;
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs

@@ -1,10 +1,9 @@
-using System;
+using Semmle.Util;
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
-using System.Security.Cryptography;
-using System.Text;
 
 namespace Semmle.BuildAnalyser
 {
@@ -19,10 +18,10 @@ namespace Semmle.BuildAnalyser
         /// Create the package manager for a specified source tree.
         /// </summary>
         /// <param name="sourceDir">The source directory.</param>
-        public NugetPackages(string sourceDir)
+        public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory)
         {
             SourceDirectory = sourceDir;
-            PackageDirectory = computeTempDirectory(sourceDir);
+            PackageDirectory = packageDirectory;
 
             // Expect nuget.exe to be in a `nuget` directory under the directory containing this exe.
             var currentAssembly = System.Reflection.Assembly.GetExecutingAssembly().Location;
@@ -50,45 +49,12 @@ namespace Semmle.BuildAnalyser
         /// </summary>
         public IEnumerable<FileInfo> PackageFiles => packages;
 
-        // Whether to delete the packages directory prior to each run.
-        // Makes each build more reproducible.
-        const bool cleanupPackages = true;
-
-        public void Cleanup(IProgressMonitor pm)
-        {
-            var packagesDirectory = new DirectoryInfo(PackageDirectory);
-
-            if (packagesDirectory.Exists)
-            {
-                try
-                {
-                    packagesDirectory.Delete(true);
-                }
-                catch (System.IO.IOException ex)
-                {
-                    pm.Warning(string.Format("Couldn't delete package directory - it's probably held open by something else: {0}", ex.Message));
-                }
-            }
-        }
-
         /// <summary>
         /// Download the packages to the temp folder.
         /// </summary>
         /// <param name="pm">The progress monitor used for reporting errors etc.</param>
         public void InstallPackages(IProgressMonitor pm)
         {
-            if (cleanupPackages)
-            {
-                Cleanup(pm);
-            }
-
-            var packagesDirectory = new DirectoryInfo(PackageDirectory);
-
-            if (!Directory.Exists(PackageDirectory))
-            {
-                packagesDirectory.Create();
-            }
-
             foreach (var package in packages)
             {
                 RestoreNugetPackage(package.FullName, pm);
@@ -109,31 +75,7 @@ namespace Semmle.BuildAnalyser
         /// This will be in the Temp location
         /// so as to not trample the source tree.
         /// </summary>
-        public string PackageDirectory
+        public TemporaryDirectory PackageDirectory { get; }
-        {
-            get;
-            private set;
-        }
-
-        readonly SHA1CryptoServiceProvider sha1 = new SHA1CryptoServiceProvider();
-
-        /// <summary>
-        /// Computes a unique temp directory for the packages associated
-        /// with this source tree. Use a SHA1 of the directory name.
-        /// </summary>
-        /// <param name="srcDir"></param>
-        /// <returns>The full path of the temp directory.</returns>
-        string computeTempDirectory(string srcDir)
-        {
-            var bytes = Encoding.Unicode.GetBytes(srcDir);
-
-            var sha = sha1.ComputeHash(bytes);
-            var sb = new StringBuilder();
-            foreach (var b in sha.Take(8))
-                sb.AppendFormat("{0:x2}", b);
-
-            return Path.Combine(Path.GetTempPath(), "Semmle", "packages", sb.ToString());
-        }
 
         /// <summary>
         /// Restore all files in a specified package.
@@ -171,16 +113,15 @@ namespace Semmle.BuildAnalyser
 
             try
             {
-                using (var p = Process.Start(pi))
+                using var p = Process.Start(pi);
-                {
-                    string output = p.StandardOutput.ReadToEnd();
-                    string error = p.StandardError.ReadToEnd();
 
-                    p.WaitForExit();
+                string output = p.StandardOutput.ReadToEnd();
-                    if (p.ExitCode != 0)
+                string error = p.StandardError.ReadToEnd();
-                    {
+
-                        pm.FailedNugetCommand(pi.FileName, pi.Arguments, output + error);
+                p.WaitForExit();
-                    }
+                if (p.ExitCode != 0)
+                {
+                    pm.FailedNugetCommand(pi.FileName, pi.Arguments, output + error);
                 }
             }
             catch (Exception ex)

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs

@@ -23,7 +23,7 @@ namespace Semmle.Extraction.CSharp.Standalone
     /// <summary>
     ///     Searches for source/references and creates separate extractions.
     /// </summary>
-    class Analysis
+    class Analysis : IDisposable
     {
         readonly ILogger logger;
 
@@ -71,12 +71,9 @@ namespace Semmle.Extraction.CSharp.Standalone
             projectExtraction.Sources.AddRange(options.SolutionFile == null ? buildAnalysis.AllSourceFiles : buildAnalysis.ProjectSourceFiles);
         }
 
-        /// <summary>
+        public void Dispose()
-        /// Delete any Nuget assemblies.
-        /// </summary>
-        public void Cleanup()
         {
-            buildAnalysis.Cleanup();
+            buildAnalysis.Dispose();
         }
     };
 
@@ -85,8 +82,9 @@ namespace Semmle.Extraction.CSharp.Standalone
         static int Main(string[] args)
         {
             var options = Options.Create(args);
+            // options.CIL = true;  // To do: Enable this
             var output = new ConsoleLogger(options.Verbosity);
-            var a = new Analysis(output);
+            using var a = new Analysis(output);
 
             if (options.Help)
             {
@@ -97,6 +95,8 @@ namespace Semmle.Extraction.CSharp.Standalone
             if (options.Errors)
                 return 1;
 
+            var start = DateTime.Now;
+
             output.Log(Severity.Info, "Running C# standalone extractor");
             a.AnalyseProjects(options);
             int sourceFiles = a.Extraction.Sources.Count();
@@ -117,10 +117,9 @@ namespace Semmle.Extraction.CSharp.Standalone
                     new ExtractionProgress(output),
                     new FileLogger(options.Verbosity, Extractor.GetCSharpLogPath()),
                     options);
-                output.Log(Severity.Info, "Extraction complete");
+                output.Log(Severity.Info, $"Extraction completed in {DateTime.Now-start}");
             }
 
-            a.Cleanup();
             return 0;
         }
 
@@ -151,7 +150,7 @@ namespace Semmle.Extraction.CSharp.Standalone
 
             public void MissingSummary(int missingTypes, int missingNamespaces)
             {
-                logger.Log(Severity.Info, "Failed to resolve {0} types and {1} namespaces", missingTypes, missingNamespaces);
+                logger.Log(Severity.Info, "Failed to resolve {0} types in {1} namespaces", missingTypes, missingNamespaces);
             }
         }
     }

csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs

@@ -1,4 +1,5 @@
 using Semmle.Util.Logging;
+using System;
 
 namespace Semmle.BuildAnalyser
 {
@@ -9,15 +10,17 @@ namespace Semmle.BuildAnalyser
     {
         void FindingFiles(string dir);
         void UnresolvedReference(string id, string project);
-        void AnalysingProjectFiles(int count);
+        void AnalysingSolution(string filename);
         void FailedProjectFile(string filename, string reason);
         void FailedNugetCommand(string exe, string args, string message);
         void NugetInstall(string package);
         void ResolvedReference(string filename);
-        void Summary(int existingSources, int usedSources, int missingSources, int references, int unresolvedReferences, int resolvedConflicts, int totalProjects, int failedProjects);
+        void Summary(int existingSources, int usedSources, int missingSources, int references, int unresolvedReferences, int resolvedConflicts, int totalProjects, int failedProjects, TimeSpan analysisTime);
         void Warning(string message);
         void ResolvedConflict(string asm1, string asm2);
         void MissingProject(string projectFile);
+        void CommandFailed(string exe, string arguments, int exitCode);
+        void MissingNuGet();
     }
 
     class ProgressMonitor : IProgressMonitor
@@ -46,9 +49,9 @@ namespace Semmle.BuildAnalyser
             logger.Log(Severity.Debug, "Unresolved {0} referenced by {1}", id, project);
         }
 
-        public void AnalysingProjectFiles(int count)
+        public void AnalysingSolution(string filename)
         {
-            logger.Log(Severity.Info, "Analyzing project files...");
+            logger.Log(Severity.Info, $"Analyzing {filename}...");
         }
 
         public void FailedProjectFile(string filename, string reason)
@@ -73,7 +76,9 @@ namespace Semmle.BuildAnalyser
         }
 
         public void Summary(int existingSources, int usedSources, int missingSources,
-            int references, int unresolvedReferences, int resolvedConflicts, int totalProjects, int failedProjects)
+            int references, int unresolvedReferences,
+            int resolvedConflicts, int totalProjects, int failedProjects,
+            TimeSpan analysisTime)
         {
             logger.Log(Severity.Info, "");
             logger.Log(Severity.Info, "Build analysis summary:");
@@ -85,6 +90,7 @@ namespace Semmle.BuildAnalyser
             logger.Log(Severity.Info, "{0, 6} resolved assembly conflicts", resolvedConflicts);
             logger.Log(Severity.Info, "{0, 6} projects", totalProjects);
             logger.Log(Severity.Info, "{0, 6} missing/failed projects", failedProjects);
+            logger.Log(Severity.Info, "Build analysis completed in {0}", analysisTime);
         }
 
         public void Warning(string message)
@@ -94,12 +100,22 @@ namespace Semmle.BuildAnalyser
 
         public void ResolvedConflict(string asm1, string asm2)
         {
-            logger.Log(Severity.Info, "Resolved {0} as {1}", asm1, asm2);
+            logger.Log(Severity.Debug, "Resolved {0} as {1}", asm1, asm2);
         }
 
         public void MissingProject(string projectFile)
         {
             logger.Log(Severity.Info, "Solution is missing {0}", projectFile);
         }
+
+        public void CommandFailed(string exe, string arguments, int exitCode)
+        {
+            logger.Log(Severity.Error, $"Command {exe} {arguments} failed with exit code {exitCode}");
+        }
+
+        public void MissingNuGet()
+        {
+            logger.Log(Severity.Error, "Missing nuget.exe");
+        }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
@@ -14,6 +14,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj" />
+    <ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
   </ItemGroup>
 
   <ItemGroup>

csharp/extractor/Semmle.Extraction.CSharp.Standalone/SolutionFile.cs

@@ -12,6 +12,8 @@ namespace Semmle.BuildAnalyser
     {
         readonly Microsoft.Build.Construction.SolutionFile solutionFile;
 
+        private string FullPath { get; }
+
         /// <summary>
         /// Read the file.
         /// </summary>
@@ -19,8 +21,8 @@ namespace Semmle.BuildAnalyser
         public SolutionFile(string filename)
         {
             // SolutionFile.Parse() expects a rooted path.
-            var fullPath = Path.GetFullPath(filename);
+            FullPath = Path.GetFullPath(filename);
-            solutionFile = Microsoft.Build.Construction.SolutionFile.Parse(fullPath);
+            solutionFile = Microsoft.Build.Construction.SolutionFile.Parse(FullPath);
         }
 
         /// <summary>

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Access.cs

@@ -45,7 +45,10 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         Access(ExpressionNodeInfo info, ISymbol symbol, bool implicitThis, IEntity target)
             : base(info.SetKind(AccessKind(info.Context, symbol)))
         {
-            cx.TrapWriter.Writer.expr_access(this, target);
+            if (!(target is null))
+            {
+                cx.TrapWriter.Writer.expr_access(this, target);
+            }
 
             if (implicitThis && !symbol.IsStatic)
             {

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MemberAccess.cs

@@ -71,7 +71,9 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
             if (symbol == null)
             {
                 info.Context.ModelError(info.Node, "Failed to determine symbol for member access");
-                return new MemberAccess(info.SetKind(ExprKind.UNKNOWN), expression, symbol);
+                // Default to property access - this can still give useful results but
+                // the target of the expression should be checked in QL.
+                return new MemberAccess(info.SetKind(ExprKind.PROPERTY_ACCESS), expression, symbol);
             }
 
             ExprKind kind;

csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs

@@ -23,7 +23,8 @@ namespace Semmle.Extraction.CSharp.Entities
 
         protected override void Populate(TextWriter trapFile)
         {
-            var ns = Namespace.Create(cx, (INamespaceSymbol)cx.GetModel(Node).GetSymbolInfo(Node.Name).Symbol);
+            var @namespace = (INamespaceSymbol) cx.GetModel(Node).GetSymbolInfo(Node.Name).Symbol;
+            var ns = Namespace.Create(cx, @namespace);
             trapFile.namespace_declarations(this, ns);
             trapFile.namespace_declaration_location(this, cx.Create(Node.Name.GetLocation()));
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs

@@ -25,7 +25,7 @@ namespace Semmle.Extraction.CSharp.Entities
         {
             if (symbol.TypeKind == TypeKind.Error)
             {
-                Context.Extractor.MissingType(symbol.ToString());
+                Context.Extractor.MissingType(symbol.ToString(), Context.FromSource);
                 return;
             }
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs

@@ -32,7 +32,7 @@ namespace Semmle.Extraction.CSharp.Entities
 
                 if (namespaceSymbol == null)
                 {
-                    cx.Extractor.MissingNamespace(Node.Name.ToFullString());
+                    cx.Extractor.MissingNamespace(Node.Name.ToFullString(), cx.FromSource);
                     cx.ModelError(Node, "Namespace not found");
                     return;
                 }

csharp/extractor/Semmle.Extraction.CSharp/SymbolExtensions.cs

@@ -214,7 +214,6 @@ namespace Semmle.Extraction.CSharp
         static void BuildNamedTypeId(this INamedTypeSymbol named, Context cx, TextWriter trapFile, Action<Context, TextWriter, ITypeSymbol> subTermAction)
         {
             bool prefixAssembly = true;
-            if (cx.Extractor.Standalone) prefixAssembly = false;
             if (named.ContainingAssembly is null) prefixAssembly = false;
 
             if (named.IsTupleType)

csharp/extractor/Semmle.Extraction/Context.cs

@@ -155,7 +155,7 @@ namespace Semmle.Extraction
 #if DEBUG_LABELS
                 using (var id = new StringWriter())
                 {
-                    entity.WriteId(id);
+                    entity.WriteQuotedId(id);
                     CheckEntityHasUniqueLabel(id.ToString(), entity);
                 }
 #endif
@@ -270,6 +270,8 @@ namespace Semmle.Extraction
             TrapWriter = trapWriter;
         }
 
+        public bool FromSource => Scope.FromSource;
+
         public bool IsGlobalContext => Scope.IsGlobalScope;
 
         public readonly ICommentGenerator CommentGenerator = new CommentProcessor();

csharp/extractor/Semmle.Extraction/ExtractionScope.cs

@@ -25,6 +25,8 @@ namespace Semmle.Extraction
         bool InFileScope(string path);
 
         bool IsGlobalScope { get; }
+
+        bool FromSource { get; }
     }
 
     /// <summary>
@@ -49,6 +51,8 @@ namespace Semmle.Extraction
         public bool InScope(ISymbol symbol) =>
             SymbolEqualityComparer.Default.Equals(symbol.ContainingAssembly, assembly) ||
             SymbolEqualityComparer.Default.Equals(symbol, assembly);
+
+        public bool FromSource => false;
     }
 
     /// <summary>
@@ -68,5 +72,7 @@ namespace Semmle.Extraction
         public bool InFileScope(string path) => path == sourceTree.FilePath;
 
         public bool InScope(ISymbol symbol) => symbol.Locations.Any(loc => loc.SourceTree == sourceTree);
+
+        public bool FromSource => true;
     }
 }

csharp/extractor/Semmle.Extraction/Extractor.cs

@@ -50,13 +50,15 @@ namespace Semmle.Extraction
         /// Record a new error type.
         /// </summary>
         /// <param name="fqn">The display name of the type, qualified where possible.</param>
-        void MissingType(string fqn);
+        /// <param name="fromSource">If the missing type was referenced from a source file.</param>
+        void MissingType(string fqn, bool fromSource);
 
         /// <summary>
         /// Record an unresolved `using namespace` directive.
         /// </summary>
         /// <param name="fqn">The full name of the namespace.</param>
-        void MissingNamespace(string fqn);
+        /// <param name="fromSource">If the missing namespace was referenced from a source file.</param>
+        void MissingNamespace(string fqn, bool fromSource);
 
         /// <summary>
         /// The list of missing types.
@@ -167,16 +169,22 @@ namespace Semmle.Extraction
         readonly ISet<string> missingTypes = new SortedSet<string>();
         readonly ISet<string> missingNamespaces = new SortedSet<string>();
 
-        public void MissingType(string fqn)
+        public void MissingType(string fqn, bool fromSource)
         {
-            lock (mutex)
+            if (fromSource)
-                missingTypes.Add(fqn);
+            {
+                lock (mutex)
+                    missingTypes.Add(fqn);
+            }
         }
 
-        public void MissingNamespace(string fqdn)
+        public void MissingNamespace(string fqdn, bool fromSource)
         {
-            lock (mutex)
+            if (fromSource)
-                missingNamespaces.Add(fqdn);
+            {
+                lock (mutex)
+                    missingNamespaces.Add(fqdn);
+            }
         }
 
         public Context CreateContext(Compilation c, TrapWriter trapWriter, IExtractionScope scope)

csharp/extractor/Semmle.Util/FileRenamer.cs

@@ -0,0 +1,34 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+
+namespace Semmle.Util
+{
+    /// <summary>
+    /// Utility to temporarily rename a set of files.
+    /// </summary>
+    public sealed class FileRenamer : IDisposable
+    {
+        readonly string[] files;
+        const string suffix = ".codeqlhidden";
+
+        public FileRenamer(IEnumerable<FileInfo> oldFiles)
+        {
+            files = oldFiles.Select(f => f.FullName).ToArray();
+
+            foreach (var file in files)
+            {
+                File.Move(file, file + suffix);
+            }
+        }
+
+        public void Dispose()
+        {
+            foreach (var file in files)
+            {
+                File.Move(file + suffix, file);
+            }
+        }
+    }
+}

csharp/extractor/Semmle.Util/TemporaryDirectory.cs

@@ -0,0 +1,30 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Security.Cryptography;
+using System.Text;
+
+namespace Semmle.Util
+{
+    /// <summary>
+    /// A temporary directory that is created within the system temp directory.
+    /// When this object is disposed, the directory is deleted.
+    /// </summary>
+    public sealed class TemporaryDirectory : IDisposable
+    {
+        public DirectoryInfo DirInfo { get; }
+
+        public TemporaryDirectory(string name)
+        {
+            DirInfo = new DirectoryInfo(name);
+            DirInfo.Create();
+        }
+
+        public void Dispose()
+        {
+            DirInfo.Delete(true);
+        }
+
+        public override string ToString() => DirInfo.FullName.ToString();
+    }
+}

csharp/ql/src/semmle/code/csharp/exprs/Access.qll

@@ -388,7 +388,12 @@ library class PropertyAccessExpr extends Expr, @property_access_expr {
   /** Gets the target of this property access. */
   Property getProperty() { expr_access(this, result) }
 
-  override string toString() { result = "access to property " + this.getProperty().getName() }
+  override string toString() {
+    result = "access to property " + this.getProperty().getName()
+    or
+    not exists(this.getProperty()) and
+    result = "access to property (unknown)"
+  }
 }
 
 /**

csharp/ql/test/library-tests/standalone/controlflow/cfg.expected

@@ -7,9 +7,14 @@
 | ControlFlow.cs:10:9:10:43 | Call (unknown target) | ControlFlow.cs:12:9:12:87 | ...; |
 | ControlFlow.cs:10:9:10:43 | call to method  | ControlFlow.cs:12:9:12:87 | ...; |
 | ControlFlow.cs:10:9:10:44 | ...; | ControlFlow.cs:10:9:10:13 | Expression |
-| ControlFlow.cs:10:22:10:22 | access to local variable v | ControlFlow.cs:10:22:10:24 | Expression |
+| ControlFlow.cs:10:22:10:22 | access to local variable v | ControlFlow.cs:10:22:10:24 | Call (unknown target) |
-| ControlFlow.cs:10:22:10:24 | Expression | ControlFlow.cs:10:22:10:26 | Expression |
+| ControlFlow.cs:10:22:10:22 | access to local variable v | ControlFlow.cs:10:22:10:24 | access to property (unknown) |
-| ControlFlow.cs:10:22:10:26 | Expression | ControlFlow.cs:10:29:10:42 | "This is true" |
+| ControlFlow.cs:10:22:10:24 | Call (unknown target) | ControlFlow.cs:10:22:10:26 | Call (unknown target) |
+| ControlFlow.cs:10:22:10:24 | Call (unknown target) | ControlFlow.cs:10:22:10:26 | access to property (unknown) |
+| ControlFlow.cs:10:22:10:24 | access to property (unknown) | ControlFlow.cs:10:22:10:26 | Call (unknown target) |
+| ControlFlow.cs:10:22:10:24 | access to property (unknown) | ControlFlow.cs:10:22:10:26 | access to property (unknown) |
+| ControlFlow.cs:10:22:10:26 | Call (unknown target) | ControlFlow.cs:10:29:10:42 | "This is true" |
+| ControlFlow.cs:10:22:10:26 | access to property (unknown) | ControlFlow.cs:10:29:10:42 | "This is true" |
 | ControlFlow.cs:10:29:10:42 | "This is true" | ControlFlow.cs:10:9:10:43 | Call (unknown target) |
 | ControlFlow.cs:10:29:10:42 | "This is true" | ControlFlow.cs:10:9:10:43 | call to method  |
 | ControlFlow.cs:12:9:12:86 | Call (unknown target) | ControlFlow.cs:12:37:12:47 | Expression |
@@ -20,5 +25,7 @@
 | ControlFlow.cs:12:51:12:62 | access to field Empty | ControlFlow.cs:12:37:12:62 | ... = ... |
 | ControlFlow.cs:12:65:12:75 | Expression | ControlFlow.cs:12:79:12:79 | access to local variable v |
 | ControlFlow.cs:12:65:12:84 | ... = ... | ControlFlow.cs:12:35:12:86 | { ..., ... } |
-| ControlFlow.cs:12:79:12:79 | access to local variable v | ControlFlow.cs:12:79:12:84 | Expression |
+| ControlFlow.cs:12:79:12:79 | access to local variable v | ControlFlow.cs:12:79:12:84 | Call (unknown target) |
-| ControlFlow.cs:12:79:12:84 | Expression | ControlFlow.cs:12:65:12:84 | ... = ... |
+| ControlFlow.cs:12:79:12:79 | access to local variable v | ControlFlow.cs:12:79:12:84 | access to property (unknown) |
+| ControlFlow.cs:12:79:12:84 | Call (unknown target) | ControlFlow.cs:12:65:12:84 | ... = ... |
+| ControlFlow.cs:12:79:12:84 | access to property (unknown) | ControlFlow.cs:12:65:12:84 | ... = ... |

csharp/ql/test/library-tests/standalone/errorrecovery/ErrorCalls.expected

@@ -2,5 +2,5 @@
 | errors.cs:43:21:43:28 | errors.cs:43:21:43:28 | object creation of type C1 | C1 |
 | errors.cs:44:13:44:19 | errors.cs:44:13:44:19 | call to method m1 | m1 |
 | errors.cs:45:13:45:19 | errors.cs:45:13:45:19 | call to method m2 | m2 |
-| errors.cs:46:13:46:38 | errors.cs:46:13:46:38 | call to method  | none |
+| errors.cs:46:13:46:38 | errors.cs:46:13:46:38 | call to method WriteLine | WriteLine |
 | errors.cs:53:17:53:25 | errors.cs:53:17:53:25 | object creation of type C2 | none |

csharp/ql/test/library-tests/standalone/regressions/ConstCase.expected

@@ -1,2 +1,3 @@
-| regressions.cs:16:13:16:37 | case ...: | regressions.cs:16:18:16:36 | Expression |
+| regressions.cs:16:13:16:37 | case ...: | regressions.cs:16:18:16:36 | access to property (unknown) |
-| regressions.cs:18:13:18:37 | case ...: | regressions.cs:18:18:18:36 | Expression |
+| regressions.cs:18:13:18:37 | case ...: | regressions.cs:18:18:18:36 | access to property (unknown) |
+| regressions.cs:20:13:20:23 | case ...: | regressions.cs:20:18:20:22 | Int32 x |

csharp/ql/test/library-tests/standalone/regressions/ConstCase.ql

@@ -1,7 +1,5 @@
 import csharp
 
 from Case c, Expr e
-where
+where e = c.getPattern().stripCasts()
-  e = c.getPattern().stripCasts() and
-  (e instanceof @unknown_expr or e instanceof ConstantPatternExpr)
 select c, e