using System; using System.IO; using System.IO.Compression; using System.Security.Cryptography; using System.Text; using Semmle.Util; using Semmle.Util.Logging; namespace Semmle.Extraction { public interface ITrapEmitter { void EmitToTrapBuilder(ITrapBuilder tb); } public sealed class TrapWriter : IDisposable { //#################### ENUMERATIONS #################### #region public enum InnerPathComputation { ABSOLUTE, RELATIVE } #endregion //#################### PRIVATE VARIABLES #################### #region /// /// The location of the src_archive directory. /// private readonly string archive; private static readonly Encoding UTF8 = new UTF8Encoding(false); private readonly bool discardDuplicates; #endregion //#################### PROPERTIES #################### #region public int IdCounter { get; set; } = 1; readonly Lazy WriterLazy; readonly Lazy BuilderLazy; TrapBuilder Builder => BuilderLazy.Value; readonly ILogger Logger; #endregion //#################### CONSTRUCTORS #################### #region public TrapWriter(ILogger logger, string outputfile, string trap, string archive, bool discardDuplicates) { Logger = logger; TrapFile = TrapPath(Logger, trap, outputfile); WriterLazy = new Lazy(() => { var tempPath = trap ?? Path.GetTempPath(); do { /* * Write the trap to a random filename in the trap folder. * Since the trap path can be very long, we need to deal with the possibility of * PathTooLongExceptions. So we use a short filename in the trap folder, * then move it later. * * Although GetRandomFileName() is cryptographically secure, * there's a tiny chance the file could already exists. */ tmpFile = Path.Combine(tempPath, Path.GetRandomFileName()); } while (File.Exists(tmpFile)); var fileStream = new FileStream(tmpFile, FileMode.CreateNew, FileAccess.Write); var compressionStream = new GZipStream(fileStream, CompressionMode.Compress); return new StreamWriter(compressionStream, UTF8, 2000000); }); BuilderLazy = new Lazy(() => new TrapBuilder(WriterLazy.Value)); this.archive = archive; this.discardDuplicates = discardDuplicates; } /// /// The output filename of the trap. /// public readonly string TrapFile; string tmpFile; // The temporary file which is moved to trapFile once written. #endregion //#################### PUBLIC METHODS #################### #region /// /// Adds the specified input file to the source archive. It may end up in either the normal or long path area /// of the source archive, depending on the length of its full path. /// /// The path to the input file. /// The encoding used by the input file. public void Archive(string inputPath, Encoding inputEncoding) { if (string.IsNullOrEmpty(archive)) return; // Calling GetFullPath makes this use the canonical capitalisation, if the file exists. string fullInputPath = Path.GetFullPath(inputPath); ArchivePath(fullInputPath, inputEncoding); } /// /// Archive a file given the file contents. /// /// The path of the file. /// The contents of the file. public void Archive(string inputPath, string contents) { if (string.IsNullOrEmpty(archive)) return; // Calling GetFullPath makes this use the canonical capitalisation, if the file exists. string fullInputPath = Path.GetFullPath(inputPath); ArchiveContents(fullInputPath, contents); } /// /// Try to move a file from sourceFile to destFile. /// If successful returns true, /// otherwise returns false and leaves the file in its original place. /// /// The source filename. /// The destination filename. /// true if the file was moved. static bool TryMove(string sourceFile, string destFile) { try { // Prefer to avoid throwing an exception if (File.Exists(destFile)) return false; File.Move(sourceFile, destFile); return true; } catch (IOException) { return false; } } /// /// Close the trap file, and move it to the right place in the trap directory. /// If the file exists already, rename it to allow the new file (ending .trap.gz) /// to sit alongside the old file (except if is true, /// in which case only the existing file is kept). /// public void Dispose() { try { if (WriterLazy.IsValueCreated) { WriterLazy.Value.Close(); if (TryMove(tmpFile, TrapFile)) return; if (discardDuplicates) { FileUtils.TryDelete(tmpFile); return; } var existingHash = ComputeHash(TrapFile); var hash = ComputeHash(tmpFile); if (existingHash != hash) { var root = TrapFile.Substring(0, TrapFile.Length - 8); // Remove trailing ".trap.gz" if (TryMove(tmpFile, $"{root}-{hash}.trap.gz")) return; } Logger.Log(Severity.Info, "Identical trap file for {0} already exists", TrapFile); FileUtils.TryDelete(tmpFile); } } catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] { Logger.Log(Severity.Error, "Failed to move the trap file from {0} to {1} because {2}", tmpFile, TrapFile, ex); } } public void Emit(ITrapEmitter emitter) { emitter.EmitToTrapBuilder(Builder); } #endregion //#################### PRIVATE METHODS #################### #region /// /// Computes the hash of . /// static string ComputeHash(string filePath) { using (var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var shaAlg = new SHA256Managed()) { var sha = shaAlg.ComputeHash(fileStream); var hex = new StringBuilder(sha.Length * 2); foreach (var b in sha) hex.AppendFormat("{0:x2}", b); return hex.ToString(); } } class TrapBuilder : ITrapBuilder { readonly StreamWriter StreamWriter; public TrapBuilder(StreamWriter sw) { StreamWriter = sw; } public ITrapBuilder Append(object arg) { StreamWriter.Write(arg); return this; } public ITrapBuilder Append(string arg) { StreamWriter.Write(arg); return this; } public ITrapBuilder AppendLine() { StreamWriter.WriteLine(); return this; } } /// /// Attempts to archive the specified input file to the normal area of the source archive. /// The file's path must be sufficiently short so as to render the path of its copy in the /// source archive less than the system path limit of 260 characters. /// /// The full path to the input file. /// The encoding used by the input file. /// If the output path in the source archive would /// exceed the system path limit of 260 characters. private void ArchivePath(string fullInputPath, Encoding inputEncoding) { string contents = File.ReadAllText(fullInputPath, inputEncoding); ArchiveContents(fullInputPath, contents); } private void ArchiveContents(string fullInputPath, string contents) { string dest = NestPaths(Logger, archive, fullInputPath, InnerPathComputation.ABSOLUTE); string tmpSrcFile = Path.GetTempFileName(); File.WriteAllText(tmpSrcFile, contents, UTF8); try { FileUtils.MoveOrReplace(tmpSrcFile, dest); } catch (IOException ex) { // If this happened, it was probably because the same file was compiled multiple times. // In any case, this is not a fatal error. Logger.Log(Severity.Warning, "Problem archiving " + dest + ": " + ex); } } public static string NestPaths(ILogger logger, string outerpath, string innerpath, InnerPathComputation innerPathComputation) { string nested = innerpath; if (!string.IsNullOrEmpty(outerpath)) { if (!Path.IsPathRooted(innerpath) && innerPathComputation == InnerPathComputation.ABSOLUTE) innerpath = Path.GetFullPath(innerpath); // Remove all leading path separators / or \ // For example, UNC paths have two leading \\ innerpath = innerpath.TrimStart(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); if (innerpath.Length > 1 && innerpath[1] == ':') innerpath = innerpath[0] + "_" + innerpath.Substring(2); nested = Path.Combine(outerpath, innerpath); } try { Directory.CreateDirectory(Path.GetDirectoryName(nested)); } catch (PathTooLongException) { logger.Log(Severity.Warning, "Failed to create parent directory of '" + nested + "': Path too long."); throw; } return nested; } public static string TrapPath(ILogger logger, string folder, string filename) { filename = Path.GetFullPath(filename) + ".trap.gz"; if (string.IsNullOrEmpty(folder)) folder = Directory.GetCurrentDirectory(); return NestPaths(logger, folder, filename, InnerPathComputation.ABSOLUTE); ; } #endregion } }