Merge pull request #391 from calumgrant/cs/extractor/dump-args

C#: Reduce extractor log size

csharp/extractor/Semmle.Autobuild.Tests/BuildScripts.cs

@@ -33,6 +33,8 @@ namespace Semmle.Extraction.Tests
             FileExistsIn.Add(file);
             if (FileExists.TryGetValue(file, out var ret))
                 return ret;
+            if (FileExists.TryGetValue(System.IO.Path.GetFileName(file), out ret))
+                return ret;
             throw new ArgumentException("Missing FileExists " + file);
         }
 

csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs

@@ -399,14 +399,6 @@ namespace Semmle.Extraction.CSharp
         /// </summary>
         public int TotalErrors => CompilationErrors + ExtractorErrors;
 
-        void AppendQuoted(StringBuilder sb, string s)
-        {
-            if (s.IndexOf(' ') != -1)
-                sb.Append('\"').Append(s).Append('\"');
-            else
-                sb.Append(s);
-        }
-
         /// <summary>
         /// Logs detailed information about this invocation,
         /// in the event that errors were detected.
@@ -414,36 +406,21 @@ namespace Semmle.Extraction.CSharp
         /// <param name="roslynArgs">The arguments passed to Roslyn.</param>
         public void LogDiagnostics(string[] roslynArgs)
         {
-            Logger.Log(Severity.Info, "  Current working directory: {0}", Directory.GetCurrentDirectory());
             Logger.Log(Severity.Info, "  Extractor: {0}", Environment.GetCommandLineArgs().First());
             if (extractor != null)
                 Logger.Log(Severity.Info, "  Extractor version: {0}", extractor.Version);
-            var sb = new StringBuilder();
-            sb.Append("  Expanded command line: ");
-            bool first = true;
-            foreach (var arg in Environment.GetCommandLineArgs().Skip(1))
-            {
-                if (arg[0] == '@')
-                {
-                    foreach (var line in File.ReadAllLines(arg.Substring(1)))
-                    {
-                        if (first) first = false;
-                        else sb.Append(" ");
-                        sb.Append(line);
-                    }
-                }
-                else
-                {
-                    if (first) first = false;
-                    else sb.Append(" ");
-                    AppendQuoted(sb, arg);
-                }
-            }
-            Logger.Log(Severity.Info, sb.ToString());
+
+            Logger.Log(Severity.Info, "  Current working directory: {0}", Directory.GetCurrentDirectory());
 
             if (roslynArgs != null)
                 Logger.Log(Severity.Info, $"  Arguments to Roslyn: {string.Join(' ', roslynArgs)}");
 
+            // Create a new file in the log folder.
+            var argsFile = Path.Combine(Extractor.GetCSharpLogDirectory(), $"csharp.{Path.GetRandomFileName()}.txt");
+
+            if (roslynArgs.ArchiveCommandLine(argsFile))
+                Logger.Log(Severity.Info, $"  Arguments have been written to {argsFile}");
+
             foreach (var error in FilteredDiagnostics)
             {
                 Logger.Log(Severity.Error, "  Compilation error: {0}", error);

csharp/extractor/Semmle.Extraction.CSharp/CompilerVersion.cs

@@ -124,7 +124,7 @@ namespace Semmle.Extraction.CSharp
         /// <returns>Modified list of arguments.</returns>
         static IEnumerable<string> AddDefaultResponse(string responseFile, IEnumerable<string> args)
         {
-            return SuppressDefaultResponseFile(args) && File.Exists(responseFile) ?
+            return SuppressDefaultResponseFile(args) || !File.Exists(responseFile) ?
                 args :
                 new[] { "@" + responseFile }.Concat(args);
         }

csharp/extractor/Semmle.Extraction.CSharp/Extractor.cs

@@ -34,8 +34,11 @@ namespace Semmle.Extraction.CSharp
 
             public void Analysed(int item, int total, string source, string output, TimeSpan time, AnalysisAction action)
             {
-                Logger.Log(Severity.Info, "  {0} -> {1} ({2})", source, output,
-                    action == AnalysisAction.Extracted ? time.ToString() : action == AnalysisAction.Excluded ? "excluded" : "up to date");
+                if (action != AnalysisAction.UpToDate)
+                {
+                    Logger.Log(Severity.Info, "  {0} ({1})", source,
+                        action == AnalysisAction.Extracted ? time.ToString() : action == AnalysisAction.Excluded ? "excluded" : "up to date");
+                }
             }
 
             public void MissingNamespace(string @namespace) { }
@@ -361,27 +364,28 @@ namespace Semmle.Extraction.CSharp
         /// <summary>
         /// Gets the path to the `csharp.log` file written to by the C# extractor.
         /// </summary>
-        public static string GetCSharpLogPath()
+        public static string GetCSharpLogPath() =>
+            Path.Combine(GetCSharpLogDirectory(), "csharp.log");
+
+        public static string GetCSharpLogDirectory()
         {
             string snapshot = Environment.GetEnvironmentVariable("ODASA_SNAPSHOT");
             string buildErrorDir = Environment.GetEnvironmentVariable("ODASA_BUILD_ERROR_DIR");
             string traps = Environment.GetEnvironmentVariable("TRAP_FOLDER");
-            string output = "csharp.log";
             if (!string.IsNullOrEmpty(snapshot))
             {
-                snapshot = Path.Combine(snapshot, "log");
-                return Path.Combine(snapshot, output);
+                return Path.Combine(snapshot, "log");
             }
             if (!string.IsNullOrEmpty(buildErrorDir))
             {
                 // Used by `qltest`
-                return Path.Combine(buildErrorDir, output);
+                return buildErrorDir;
             }
             if (!string.IsNullOrEmpty(traps))
             {
-                return Path.Combine(traps, output);
+                return traps;
             }
-            return output;
+            return Directory.GetCurrentDirectory();
         }
     }
 }

csharp/extractor/Semmle.Extraction.Tests/Options.cs

@@ -1,6 +1,8 @@
 using Xunit;
 using Semmle.Util.Logging;
 using System;
+using System.IO;
+using Semmle.Util;
 
 namespace Semmle.Extraction.Tests
 {
@@ -184,5 +186,24 @@ namespace Semmle.Extraction.Tests
             options = CSharp.Options.CreateWithEnvironment(new string[] {});
             Assert.True(options.Fast);
         }
+
+        [Fact]
+        public void ArchiveArguments()
+        {
+            var file1 = Path.GetTempFileName();
+            var file2 = Path.GetTempFileName();
+
+            try
+            {
+                File.AppendAllText(file1, "Test");
+                new string[] { "/noconfig", "@" + file1 }.ArchiveCommandLine(file2);
+                Assert.Equal("Test", File.ReadAllText(file2));
+            }
+            finally
+            {
+                File.Delete(file1);
+                File.Delete(file2);
+            }
+        }
     }
 }

csharp/extractor/Semmle.Extraction/Message.cs

@@ -14,5 +14,7 @@ namespace Semmle.Extraction
         public ISymbol symbol;
         public SyntaxNode node;
         public Exception exception;
+
+        public override string ToString() => message;
     }
 }

csharp/extractor/Semmle.Util/CommandLineExtensions.cs

@@ -0,0 +1,26 @@
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+
+namespace Semmle.Util
+{
+    public static class CommandLineExtensions
+    {
+        /// <summary>
+        /// Archives the first "@" argument in a list of command line arguments.
+        /// Subsequent "@" arguments are ignored.
+        /// </summary>
+        /// <param name="commandLineArguments">The raw command line arguments.</param>
+        /// <param name="filename">The full filename to write to.</param>
+        /// <returns>True iff the file was written.</returns>
+        public static bool ArchiveCommandLine(this IEnumerable<string> commandLineArguments, string filename)
+        {
+            foreach (var arg in commandLineArguments.Where(arg => arg[0] == '@').Select(arg => arg.Substring(1)))
+            {
+                File.Copy(arg, filename, true);
+                return true;
+            }
+            return false;
+        }
+    }
+}