mirror of
https://github.com/github/codeql.git
synced 2026-05-01 11:45:14 +02:00
Move more classes to the Semmle.Extraction.CSharp namespace
This commit is contained in:
Tamas Vajk
@@ -0,0 +1,25 @@
|
||||
using Microsoft.CodeAnalysis;
|
||||
|
||||
namespace Semmle.Extraction.CSharp
|
||||
{
|
||||
/// <summary>
|
||||
/// The scope of symbols in an assembly.
|
||||
/// </summary>
|
||||
public class AssemblyScope : IExtractionScope
|
||||
{
|
||||
private readonly IAssemblySymbol assembly;
|
||||
private readonly string filepath;
|
||||
|
||||
public AssemblyScope(IAssemblySymbol symbol, string path)
|
||||
{
|
||||
assembly = symbol;
|
||||
filepath = path;
|
||||
}
|
||||
|
||||
public bool InFileScope(string path) => path == filepath;
|
||||
|
||||
public bool InScope(ISymbol symbol) =>
|
||||
SymbolEqualityComparer.Default.Equals(symbol.ContainingAssembly, assembly) ||
|
||||
SymbolEqualityComparer.Default.Equals(symbol, assembly);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
using System;
|
||||
using Semmle.Util;
|
||||
using Semmle.Util.Logging;
|
||||
|
||||
namespace Semmle.Extraction.CSharp
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents the parsed state of the command line arguments.
|
||||
/// This represents the common options.
|
||||
/// </summary>
|
||||
public abstract class CommonOptions : ICommandLineOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// The specified number of threads, or the default if unspecified.
|
||||
/// </summary>
|
||||
public int Threads { get; private set; } = EnvironmentVariables.GetDefaultNumberOfThreads();
|
||||
|
||||
/// <summary>
|
||||
/// The verbosity used specified by the '--silent' or '--verbose' flags or the '--verbosity' option.
|
||||
/// </summary>
|
||||
public Verbosity LegacyVerbosity { get; protected set; } = Verbosity.Info;
|
||||
|
||||
private Verbosity? verbosity = null;
|
||||
public Verbosity Verbosity
|
||||
{
|
||||
get
|
||||
{
|
||||
if (verbosity != null)
|
||||
{
|
||||
return verbosity.Value;
|
||||
}
|
||||
|
||||
var envVarValue = EnvironmentVariables.GetExtractorOption("LOGGING_VERBOSITY");
|
||||
verbosity = VerbosityExtensions.ParseVerbosity(envVarValue, logThreadId: true);
|
||||
if (verbosity != null)
|
||||
{
|
||||
return verbosity.Value;
|
||||
}
|
||||
|
||||
envVarValue = Environment.GetEnvironmentVariable("CODEQL_VERBOSITY");
|
||||
verbosity = VerbosityExtensions.ParseVerbosity(envVarValue, logThreadId: true);
|
||||
if (verbosity != null)
|
||||
{
|
||||
return verbosity.Value;
|
||||
}
|
||||
|
||||
// This only works, because we already parsed the provided options, so `LegacyVerbosity` is already set (or it still has the default value).
|
||||
verbosity = LegacyVerbosity;
|
||||
return verbosity.Value;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether to output to the console.
|
||||
/// </summary>
|
||||
public bool Console { get; private set; } = false;
|
||||
|
||||
/// <summary>
|
||||
/// Holds if assemblies shouldn't be extracted twice.
|
||||
/// </summary>
|
||||
public bool Cache { get; private set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Whether extraction is done using `codeql test run`.
|
||||
/// </summary>
|
||||
public bool QlTest { get; private set; } = false;
|
||||
|
||||
/// <summary>
|
||||
/// The compression algorithm used for trap files.
|
||||
/// </summary>
|
||||
public TrapWriter.CompressionMode TrapCompression { get; private set; } = TrapWriter.CompressionMode.Brotli;
|
||||
|
||||
public virtual bool HandleOption(string key, string value)
|
||||
{
|
||||
switch (key)
|
||||
{
|
||||
case "threads":
|
||||
Threads = int.Parse(value);
|
||||
return true;
|
||||
case "verbosity":
|
||||
LegacyVerbosity = (Verbosity)int.Parse(value);
|
||||
return true;
|
||||
case "trap_compression":
|
||||
if (Enum.TryParse<TrapWriter.CompressionMode>(value, true, out var mode))
|
||||
{
|
||||
TrapCompression = mode;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public abstract bool HandleArgument(string argument);
|
||||
|
||||
public virtual bool HandleFlag(string flag, bool value)
|
||||
{
|
||||
switch (flag)
|
||||
{
|
||||
case "silent":
|
||||
LegacyVerbosity = value ? Verbosity.Off : Verbosity.Info;
|
||||
return true;
|
||||
case "verbose":
|
||||
LegacyVerbosity = value ? Verbosity.Debug : Verbosity.Error;
|
||||
return true;
|
||||
case "console":
|
||||
Console = value;
|
||||
return true;
|
||||
case "cache":
|
||||
Cache = value;
|
||||
return true;
|
||||
case "qltest":
|
||||
QlTest = value;
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public abstract void InvalidArgument(string argument);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Xml;
|
||||
|
||||
namespace Semmle.Extraction.CSharp
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents a .csproj file and reads information from it.
|
||||
/// </summary>
|
||||
public class CsProjFile
|
||||
{
|
||||
private string Filename { get; }
|
||||
|
||||
private string Directory { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Reads the .csproj file.
|
||||
/// </summary>
|
||||
/// <param name="filename">The .csproj file.</param>
|
||||
public CsProjFile(FileInfo filename)
|
||||
{
|
||||
Filename = filename.FullName;
|
||||
|
||||
var directoryName = Path.GetDirectoryName(Filename);
|
||||
|
||||
if (directoryName is null)
|
||||
{
|
||||
throw new InternalError($"Directory of file '{Filename}' is null");
|
||||
}
|
||||
|
||||
Directory = directoryName;
|
||||
|
||||
try
|
||||
{
|
||||
// This can fail if the .csproj is invalid or has
|
||||
// unrecognised content or is the wrong version.
|
||||
// This currently always fails on Linux because
|
||||
// Microsoft.Build is not cross platform.
|
||||
(csFiles, references, projectReferences) = ReadMsBuildProject(filename);
|
||||
}
|
||||
catch // lgtm[cs/catch-of-all-exceptions]
|
||||
{
|
||||
// There was some reason why the project couldn't be loaded.
|
||||
// Fall back to reading the Xml document directly.
|
||||
// This method however doesn't handle variable expansion.
|
||||
(csFiles, references, projectReferences) = ReadProjectFileAsXml(filename, Directory);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Read the .csproj file using Microsoft Build.
|
||||
/// This occasionally fails if the project file is incompatible for some reason,
|
||||
/// and there seems to be no way to make it succeed. Fails on Linux.
|
||||
/// </summary>
|
||||
/// <param name="filename">The file to read.</param>
|
||||
private static (string[] csFiles, string[] references, string[] projectReferences) ReadMsBuildProject(FileInfo filename)
|
||||
{
|
||||
var msbuildProject = new Microsoft.Build.Execution.ProjectInstance(filename.FullName);
|
||||
|
||||
var references = msbuildProject.Items
|
||||
.Where(item => item.ItemType == "Reference")
|
||||
.Select(item => item.EvaluatedInclude)
|
||||
.ToArray();
|
||||
|
||||
var projectReferences = msbuildProject.Items
|
||||
.Where(item => item.ItemType == "ProjectReference")
|
||||
.Select(item => item.EvaluatedInclude)
|
||||
.ToArray();
|
||||
|
||||
var csFiles = msbuildProject.Items
|
||||
.Where(item => item.ItemType == "Compile")
|
||||
.Select(item => item.GetMetadataValue("FullPath"))
|
||||
.Where(fn => fn.EndsWith(".cs"))
|
||||
.ToArray();
|
||||
|
||||
return (csFiles, references, projectReferences);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the .csproj file directly as XML.
|
||||
/// This doesn't handle variables etc, and should only used as a
|
||||
/// fallback if ReadMsBuildProject() fails.
|
||||
/// </summary>
|
||||
/// <param name="fileName">The .csproj file.</param>
|
||||
private static (string[] csFiles, string[] references, string[] projectReferences) ReadProjectFileAsXml(FileInfo fileName, string directoryName)
|
||||
{
|
||||
var projFile = new XmlDocument();
|
||||
var mgr = new XmlNamespaceManager(projFile.NameTable);
|
||||
mgr.AddNamespace("msbuild", "http://schemas.microsoft.com/developer/msbuild/2003");
|
||||
projFile.Load(fileName.FullName);
|
||||
var projDir = fileName.Directory;
|
||||
var root = projFile.DocumentElement;
|
||||
|
||||
if (root is null)
|
||||
{
|
||||
throw new NotSupportedException("Project file without root is not supported.");
|
||||
}
|
||||
|
||||
// Figure out if it's dotnet core
|
||||
|
||||
var netCoreProjectFile = root.GetAttribute("Sdk").StartsWith("Microsoft.NET.Sdk");
|
||||
|
||||
if (netCoreProjectFile)
|
||||
{
|
||||
var explicitCsFiles = root
|
||||
.SelectNodes("/Project/ItemGroup/Compile/@Include", mgr)
|
||||
?.NodeList()
|
||||
.Select(node => node.Value)
|
||||
.Select(cs => GetFullPath(cs, projDir))
|
||||
.Where(s => s is not null)
|
||||
?? Enumerable.Empty<string>();
|
||||
|
||||
var additionalCsFiles = System.IO.Directory.GetFiles(directoryName, "*.cs", new EnumerationOptions { RecurseSubdirectories = true, MatchCasing = MatchCasing.CaseInsensitive });
|
||||
|
||||
var projectReferences = root
|
||||
.SelectNodes("/Project/ItemGroup/ProjectReference/@Include", mgr)
|
||||
?.NodeList()
|
||||
.Select(node => node.Value)
|
||||
.Select(csproj => GetFullPath(csproj, projDir))
|
||||
.Where(s => s is not null)
|
||||
?? Enumerable.Empty<string>();
|
||||
|
||||
#nullable disable warnings
|
||||
return (explicitCsFiles.Concat(additionalCsFiles).ToArray(), Array.Empty<string>(), projectReferences.ToArray());
|
||||
#nullable restore warnings
|
||||
}
|
||||
|
||||
var references = root
|
||||
.SelectNodes("/msbuild:Project/msbuild:ItemGroup/msbuild:Reference/@Include", mgr)
|
||||
?.NodeList()
|
||||
.Select(node => node.Value)
|
||||
.Where(s => s is not null)
|
||||
.ToArray()
|
||||
?? Array.Empty<string>();
|
||||
|
||||
var relativeCsIncludes = root
|
||||
.SelectNodes("/msbuild:Project/msbuild:ItemGroup/msbuild:Compile/@Include", mgr)
|
||||
?.NodeList()
|
||||
.Select(node => node.Value)
|
||||
.ToArray()
|
||||
?? Array.Empty<string>();
|
||||
|
||||
var csFiles = relativeCsIncludes
|
||||
.Select(cs => GetFullPath(cs, projDir))
|
||||
.Where(s => s is not null)
|
||||
.ToArray();
|
||||
|
||||
#nullable disable warnings
|
||||
return (csFiles, references, Array.Empty<string>());
|
||||
#nullable restore warnings
|
||||
}
|
||||
|
||||
private static string? GetFullPath(string? file, DirectoryInfo? projDir)
|
||||
{
|
||||
if (file is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return Path.GetFullPath(Path.Combine(projDir?.FullName ?? string.Empty, Path.DirectorySeparatorChar == '/' ? file.Replace("\\", "/") : file));
|
||||
}
|
||||
|
||||
private readonly string[] references;
|
||||
private readonly string[] projectReferences;
|
||||
private readonly string[] csFiles;
|
||||
|
||||
/// <summary>
|
||||
/// The list of references as a list of assembly IDs.
|
||||
/// </summary>
|
||||
public IEnumerable<string> References => references;
|
||||
|
||||
/// <summary>
|
||||
/// The list of project references in full path format.
|
||||
/// </summary>
|
||||
public IEnumerable<string> ProjectReferences => projectReferences;
|
||||
|
||||
/// <summary>
|
||||
/// The list of C# source files in full path format.
|
||||
/// </summary>
|
||||
public IEnumerable<string> Sources => csFiles;
|
||||
}
|
||||
|
||||
internal static class XmlNodeHelper
|
||||
{
|
||||
/// <summary>
|
||||
/// Helper to convert an XmlNodeList into an IEnumerable.
|
||||
/// This allows it to be used with Linq.
|
||||
/// </summary>
|
||||
/// <param name="list">The list to convert.</param>
|
||||
/// <returns>A more useful data type.</returns>
|
||||
public static IEnumerable<XmlNode> NodeList(this XmlNodeList list)
|
||||
{
|
||||
return list.OfType<XmlNode>();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using Semmle.Util;
|
||||
|
||||
namespace Semmle.Extraction.CSharp
|
||||
{
|
||||
public sealed class InvalidFilePatternException : Exception
|
||||
{
|
||||
public InvalidFilePatternException(string pattern, string message) :
|
||||
base($"Invalid file pattern '{pattern}': {message}")
|
||||
{ }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A file pattern, as used in either an extractor layout file or
|
||||
/// a path transformer file.
|
||||
/// </summary>
|
||||
public sealed class FilePattern
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether this is an inclusion pattern.
|
||||
/// </summary>
|
||||
public bool Include { get; }
|
||||
|
||||
public FilePattern(string pattern)
|
||||
{
|
||||
Include = true;
|
||||
if (pattern.StartsWith("-"))
|
||||
{
|
||||
pattern = pattern.Substring(1);
|
||||
Include = false;
|
||||
}
|
||||
pattern = FileUtils.ConvertToUnix(pattern.Trim()).TrimStart('/');
|
||||
RegexPattern = BuildRegex(pattern).ToString();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Constructs a regex string from a file pattern. Throws
|
||||
/// `InvalidFilePatternException` for invalid patterns.
|
||||
/// </summary>
|
||||
private static StringBuilder BuildRegex(string pattern)
|
||||
{
|
||||
bool HasCharAt(int i, Predicate<char> p) =>
|
||||
i >= 0 && i < pattern.Length && p(pattern[i]);
|
||||
var sb = new StringBuilder();
|
||||
var i = 0;
|
||||
var seenDoubleSlash = false;
|
||||
sb.Append('^');
|
||||
while (i < pattern.Length)
|
||||
{
|
||||
if (pattern[i] == '/')
|
||||
{
|
||||
if (HasCharAt(i + 1, c => c == '/'))
|
||||
{
|
||||
if (seenDoubleSlash)
|
||||
throw new InvalidFilePatternException(pattern, "'//' is allowed at most once.");
|
||||
sb.Append("(?<doubleslash>/)");
|
||||
i += 2;
|
||||
seenDoubleSlash = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.Append('/');
|
||||
i++;
|
||||
}
|
||||
}
|
||||
else if (pattern[i] == '*')
|
||||
{
|
||||
if (HasCharAt(i + 1, c => c == '*'))
|
||||
{
|
||||
if (HasCharAt(i - 1, c => c != '/'))
|
||||
throw new InvalidFilePatternException(pattern, "'**' preceeded by non-`/` character.");
|
||||
if (HasCharAt(i + 2, c => c != '/'))
|
||||
throw new InvalidFilePatternException(pattern, "'**' succeeded by non-`/` character");
|
||||
|
||||
if (i + 2 < pattern.Length)
|
||||
{
|
||||
// Processing .../**/...
|
||||
// ^^^
|
||||
sb.Append("(.*/|)");
|
||||
i += 3;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Processing .../** at the end of the pattern.
|
||||
// There's no need to add .* because it's anyways added outside the loop.
|
||||
break;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.Append("[^/]*");
|
||||
i++;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.Append(Regex.Escape(pattern[i++].ToString()));
|
||||
}
|
||||
}
|
||||
return sb.Append(".*");
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// The regex pattern compiled from this file pattern.
|
||||
/// </summary>
|
||||
public string RegexPattern { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Returns `true` if the set of file patterns `patterns` match the path `path`.
|
||||
/// If so, `transformerSuffix` will contain the part of `path` that needs to be
|
||||
/// suffixed when using path transformers.
|
||||
/// </summary>
|
||||
public static bool Matches(IEnumerable<FilePattern> patterns, string path, [NotNullWhen(true)] out string? transformerSuffix)
|
||||
{
|
||||
path = FileUtils.ConvertToUnix(path).TrimStart('/');
|
||||
|
||||
foreach (var pattern in patterns.Reverse())
|
||||
{
|
||||
var m = new Regex(pattern.RegexPattern).Match(path);
|
||||
if (m.Success)
|
||||
{
|
||||
if (pattern.Include)
|
||||
{
|
||||
transformerSuffix = m.Groups.TryGetValue("doubleslash", out var group)
|
||||
? path.Substring(group.Index)
|
||||
: path;
|
||||
return true;
|
||||
}
|
||||
|
||||
transformerSuffix = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
transformerSuffix = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
using Microsoft.CodeAnalysis;
|
||||
|
||||
namespace Semmle.Extraction.CSharp
|
||||
{
|
||||
/// <summary>
|
||||
/// Defines which entities belong in the trap file
|
||||
/// for the currently extracted entity. This is used to ensure that
|
||||
/// trap files do not contain redundant information. Generally a symbol
|
||||
/// should have an affinity with exactly one trap file, except for constructed
|
||||
/// symbols.
|
||||
/// </summary>
|
||||
public interface IExtractionScope
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether the given symbol belongs in the trap file.
|
||||
/// </summary>
|
||||
/// <param name="symbol">The symbol to populate.</param>
|
||||
bool InScope(ISymbol symbol);
|
||||
|
||||
/// <summary>
|
||||
/// Whether the given file belongs in the trap file.
|
||||
/// </summary>
|
||||
/// <param name="path">The path to populate.</param>
|
||||
bool InFileScope(string path);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.IO;
|
||||
using Semmle.Util;
|
||||
using Semmle.Util.Logging;
|
||||
|
||||
namespace Semmle.Extraction.CSharp
|
||||
{
|
||||
/// <summary>
|
||||
/// A class for interpreting path transformers specified using the environment
|
||||
/// variable `CODEQL_PATH_TRANSFORMER`.
|
||||
/// </summary>
|
||||
public sealed class PathTransformer
|
||||
{
|
||||
public class InvalidPathTransformerException : Exception
|
||||
{
|
||||
public InvalidPathTransformerException(string message) :
|
||||
base($"Invalid path transformer specification: {message}")
|
||||
{ }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A transformed path.
|
||||
/// </summary>
|
||||
public interface ITransformedPath
|
||||
{
|
||||
string Value { get; }
|
||||
|
||||
string Extension { get; }
|
||||
|
||||
string NameWithoutExtension { get; }
|
||||
|
||||
ITransformedPath? ParentDirectory { get; }
|
||||
|
||||
ITransformedPath WithSuffix(string suffix);
|
||||
|
||||
string DatabaseId { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the name of the trap file for this file.
|
||||
/// </summary>
|
||||
/// <returns>The full filepath of the trap file.</returns>
|
||||
public string GetTrapPath(ILogger logger, TrapWriter.CompressionMode trapCompression) =>
|
||||
TrapWriter.TrapPath(logger, Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"), this, trapCompression);
|
||||
|
||||
/// <summary>
|
||||
/// Creates a trap writer for this file.
|
||||
/// </summary>
|
||||
/// <returns>A newly created TrapWriter.</returns>
|
||||
public TrapWriter CreateTrapWriter(ILogger logger, TrapWriter.CompressionMode trapCompression, bool discardDuplicates) =>
|
||||
new(logger, this, Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"), Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"), trapCompression, discardDuplicates);
|
||||
}
|
||||
|
||||
private struct TransformedPath : ITransformedPath
|
||||
{
|
||||
public TransformedPath(string value) { this.value = value; }
|
||||
private readonly string value;
|
||||
|
||||
public string Value => value;
|
||||
|
||||
public string Extension
|
||||
{
|
||||
get
|
||||
{
|
||||
var extension = Path.GetExtension(value);
|
||||
return string.IsNullOrEmpty(extension) ? "" : extension.Substring(1);
|
||||
}
|
||||
}
|
||||
|
||||
public string NameWithoutExtension => Path.GetFileNameWithoutExtension(value);
|
||||
|
||||
public ITransformedPath? ParentDirectory
|
||||
{
|
||||
get
|
||||
{
|
||||
var dir = Path.GetDirectoryName(value);
|
||||
if (dir is null)
|
||||
return null;
|
||||
var isWindowsDriveLetter = dir.Length == 2 && char.IsLetter(dir[0]) && dir[1] == ':';
|
||||
if (isWindowsDriveLetter)
|
||||
return null;
|
||||
return new TransformedPath(FileUtils.ConvertToUnix(dir));
|
||||
}
|
||||
}
|
||||
|
||||
public ITransformedPath WithSuffix(string suffix) => new TransformedPath(value + suffix);
|
||||
|
||||
public string DatabaseId
|
||||
{
|
||||
get
|
||||
{
|
||||
var ret = value;
|
||||
if (ret.Length >= 2 && ret[1] == ':' && Char.IsLower(ret[0]))
|
||||
ret = $"{char.ToUpper(ret[0])}_{ret[2..]}";
|
||||
return ret.Replace('\\', '/').Replace(":", "_");
|
||||
}
|
||||
}
|
||||
|
||||
public override int GetHashCode() => 11 * value.GetHashCode();
|
||||
|
||||
public override bool Equals(object? obj) => obj is TransformedPath tp && tp.value == value;
|
||||
|
||||
public override string ToString() => value;
|
||||
}
|
||||
|
||||
private readonly Func<string, string> transform;
|
||||
|
||||
/// <summary>
|
||||
/// Returns the path obtained by transforming `path`.
|
||||
/// </summary>
|
||||
public ITransformedPath Transform(string path) => new TransformedPath(transform(path));
|
||||
|
||||
/// <summary>
|
||||
/// Default constructor reads parameters from the environment.
|
||||
/// </summary>
|
||||
public PathTransformer(IPathCache pathCache) :
|
||||
this(pathCache, Environment.GetEnvironmentVariable("CODEQL_PATH_TRANSFORMER") is string file ? File.ReadAllLines(file) : null)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a path transformer based on the specification in `lines`.
|
||||
/// Throws `InvalidPathTransformerException` for invalid specifications.
|
||||
/// </summary>
|
||||
public PathTransformer(IPathCache pathCache, string[]? lines)
|
||||
{
|
||||
if (lines is null)
|
||||
{
|
||||
transform = path => FileUtils.ConvertToUnix(pathCache.GetCanonicalPath(path));
|
||||
return;
|
||||
}
|
||||
|
||||
var sections = ParsePathTransformerSpec(lines);
|
||||
transform = path =>
|
||||
{
|
||||
path = FileUtils.ConvertToUnix(pathCache.GetCanonicalPath(path));
|
||||
foreach (var section in sections)
|
||||
{
|
||||
if (section.Matches(path, out var transformed))
|
||||
return transformed;
|
||||
}
|
||||
return path;
|
||||
};
|
||||
}
|
||||
|
||||
private static IEnumerable<TransformerSection> ParsePathTransformerSpec(string[] lines)
|
||||
{
|
||||
var sections = new List<TransformerSection>();
|
||||
try
|
||||
{
|
||||
var i = 0;
|
||||
while (i < lines.Length && !lines[i].StartsWith("#"))
|
||||
i++;
|
||||
while (i < lines.Length)
|
||||
{
|
||||
var section = new TransformerSection(lines, ref i);
|
||||
sections.Add(section);
|
||||
}
|
||||
|
||||
if (sections.Count == 0)
|
||||
throw new InvalidPathTransformerException("contains no sections.");
|
||||
}
|
||||
catch (InvalidFilePatternException ex)
|
||||
{
|
||||
throw new InvalidPathTransformerException(ex.Message);
|
||||
}
|
||||
return sections;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class TransformerSection
|
||||
{
|
||||
private readonly string name;
|
||||
private readonly List<FilePattern> filePatterns = new List<FilePattern>();
|
||||
|
||||
public TransformerSection(string[] lines, ref int i)
|
||||
{
|
||||
name = lines[i++].Substring(1); // skip the '#'
|
||||
for (; i < lines.Length && !lines[i].StartsWith("#"); i++)
|
||||
{
|
||||
var line = lines[i];
|
||||
if (!string.IsNullOrWhiteSpace(line))
|
||||
filePatterns.Add(new FilePattern(line));
|
||||
}
|
||||
}
|
||||
|
||||
public bool Matches(string path, [NotNullWhen(true)] out string? transformed)
|
||||
{
|
||||
if (FilePattern.Matches(filePatterns, path, out var suffix))
|
||||
{
|
||||
transformed = FileUtils.ConvertToUnix(name) + suffix;
|
||||
return true;
|
||||
}
|
||||
transformed = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
using System.Linq;
|
||||
using Microsoft.CodeAnalysis;
|
||||
|
||||
|
||||
namespace Semmle.Extraction.CSharp
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// The scope of symbols in a source file.
|
||||
/// </summary>
|
||||
public class SourceScope : IExtractionScope
|
||||
{
|
||||
public SyntaxTree SourceTree { get; }
|
||||
|
||||
public SourceScope(SyntaxTree tree)
|
||||
{
|
||||
SourceTree = tree;
|
||||
}
|
||||
|
||||
public bool InFileScope(string path) => path == SourceTree.FilePath;
|
||||
|
||||
public bool InScope(ISymbol symbol) => symbol.Locations.Any(loc => loc.SourceTree == SourceTree);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Text;
|
||||
using Semmle.Util;
|
||||
using Semmle.Util.Logging;
|
||||
|
||||
namespace Semmle.Extraction.CSharp
|
||||
{
|
||||
public interface ITrapEmitter
|
||||
{
|
||||
void EmitTrap(TextWriter trapFile);
|
||||
}
|
||||
|
||||
public sealed class TrapWriter : IDisposable
|
||||
{
|
||||
public enum CompressionMode
|
||||
{
|
||||
None,
|
||||
Gzip,
|
||||
Brotli
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The location of the src_archive directory.
|
||||
/// </summary>
|
||||
private readonly string? archive;
|
||||
private static readonly Encoding utf8 = new UTF8Encoding(false);
|
||||
|
||||
private readonly bool discardDuplicates;
|
||||
|
||||
public int IdCounter { get; set; } = 1;
|
||||
|
||||
private readonly Lazy<StreamWriter> writerLazy;
|
||||
|
||||
public StreamWriter Writer => writerLazy.Value;
|
||||
|
||||
private readonly ILogger logger;
|
||||
|
||||
private readonly CompressionMode trapCompression;
|
||||
|
||||
public TrapWriter(ILogger logger, PathTransformer.ITransformedPath outputfile, string? trap, string? archive, CompressionMode trapCompression, bool discardDuplicates)
|
||||
{
|
||||
this.logger = logger;
|
||||
this.trapCompression = trapCompression;
|
||||
|
||||
TrapFile = TrapPath(this.logger, trap, outputfile, trapCompression);
|
||||
|
||||
writerLazy = new Lazy<StreamWriter>(() =>
|
||||
{
|
||||
var tempPath = trap ?? FileUtils.GetTemporaryWorkingDirectory(out _);
|
||||
|
||||
do
|
||||
{
|
||||
/*
|
||||
* Write the trap to a random filename in the trap folder.
|
||||
* Since the trap path can be very long, we need to deal with the possibility of
|
||||
* PathTooLongExceptions. So we use a short filename in the trap folder,
|
||||
* then move it later.
|
||||
*
|
||||
* Although GetRandomFileName() is cryptographically secure,
|
||||
* there's a tiny chance the file could already exists.
|
||||
*/
|
||||
tmpFile = Path.Combine(tempPath, Path.GetRandomFileName());
|
||||
}
|
||||
while (File.Exists(tmpFile));
|
||||
|
||||
var fileStream = new FileStream(tmpFile, FileMode.CreateNew, FileAccess.Write);
|
||||
|
||||
Stream compressionStream;
|
||||
|
||||
switch (trapCompression)
|
||||
{
|
||||
case CompressionMode.Brotli:
|
||||
compressionStream = new BrotliStream(fileStream, CompressionLevel.Fastest);
|
||||
break;
|
||||
case CompressionMode.Gzip:
|
||||
compressionStream = new GZipStream(fileStream, CompressionLevel.Fastest);
|
||||
break;
|
||||
case CompressionMode.None:
|
||||
compressionStream = fileStream;
|
||||
break;
|
||||
default:
|
||||
throw new ArgumentOutOfRangeException(nameof(trapCompression), trapCompression, "Unsupported compression type");
|
||||
}
|
||||
|
||||
|
||||
return new StreamWriter(compressionStream, utf8, 2000000);
|
||||
});
|
||||
this.archive = archive;
|
||||
this.discardDuplicates = discardDuplicates;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The output filename of the trap.
|
||||
/// </summary>
|
||||
public string TrapFile { get; }
|
||||
private string tmpFile = ""; // The temporary file which is moved to trapFile once written.
|
||||
|
||||
/// <summary>
|
||||
/// Adds the specified input file to the source archive. It may end up in either the normal or long path area
|
||||
/// of the source archive, depending on the length of its full path.
|
||||
/// </summary>
|
||||
/// <param name="originalPath">The path to the input file.</param>
|
||||
/// <param name="transformedPath">The transformed path to the input file.</param>
|
||||
/// <param name="inputEncoding">The encoding used by the input file.</param>
|
||||
public void Archive(string originalPath, PathTransformer.ITransformedPath transformedPath, Encoding inputEncoding)
|
||||
{
|
||||
Archive(() =>
|
||||
{
|
||||
var fullInputPath = Path.GetFullPath(originalPath);
|
||||
return File.ReadAllText(fullInputPath, inputEncoding);
|
||||
}, transformedPath);
|
||||
}
|
||||
|
||||
public void ArchiveContent(string contents, PathTransformer.ITransformedPath transformedPath)
|
||||
{
|
||||
Archive(() => contents, transformedPath);
|
||||
}
|
||||
|
||||
private void Archive(Func<string> getContent, PathTransformer.ITransformedPath transformedPath)
|
||||
{
|
||||
if (string.IsNullOrEmpty(archive))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var dest = FileUtils.NestPaths(logger, archive, transformedPath.Value);
|
||||
try
|
||||
{
|
||||
var tmpSrcFile = Path.GetTempFileName();
|
||||
File.WriteAllText(tmpSrcFile, getContent(), utf8);
|
||||
|
||||
FileUtils.MoveOrReplace(tmpSrcFile, dest);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// If this happened, it was probably because
|
||||
// - the same file was compiled multiple times, or
|
||||
// - the file doesn't exist (due to wrong #line directive or because it's an in-memory source generated AST).
|
||||
// In any case, this is not a fatal error.
|
||||
logger.LogWarning($"Problem archiving {dest}: {ex}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Try to move a file from sourceFile to destFile.
|
||||
/// If successful returns true,
|
||||
/// otherwise returns false and leaves the file in its original place.
|
||||
/// </summary>
|
||||
/// <param name="sourceFile">The source filename.</param>
|
||||
/// <param name="destFile">The destination filename.</param>
|
||||
/// <returns>true if the file was moved.</returns>
|
||||
private static bool TryMove(string sourceFile, string destFile)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Prefer to avoid throwing an exception
|
||||
if (File.Exists(destFile))
|
||||
return false;
|
||||
|
||||
File.Move(sourceFile, destFile);
|
||||
return true;
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Close the trap file, and move it to the right place in the trap directory.
|
||||
/// If the file exists already, rename it to allow the new file (ending .trap.gz)
|
||||
/// to sit alongside the old file (except if <paramref name="discardDuplicates"/> is true,
|
||||
/// in which case only the existing file is kept).
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (writerLazy.IsValueCreated)
|
||||
{
|
||||
writerLazy.Value.Close();
|
||||
if (TryMove(tmpFile, TrapFile))
|
||||
return;
|
||||
|
||||
if (discardDuplicates)
|
||||
{
|
||||
FileUtils.TryDelete(tmpFile);
|
||||
return;
|
||||
}
|
||||
|
||||
var existingHash = FileUtils.ComputeFileHash(TrapFile);
|
||||
var hash = FileUtils.ComputeFileHash(tmpFile);
|
||||
if (existingHash != hash)
|
||||
{
|
||||
var extension = TrapExtension(trapCompression);
|
||||
var root = TrapFile[..^extension.Length]; // Remove trailing ".trap", ".trap.gz", or ".trap.br"
|
||||
var newTrapName = $"{root}-{hash}{extension}";
|
||||
logger.LogInfo($"Identical trap file for {TrapFile} already exists, renaming to {newTrapName}");
|
||||
if (TryMove(tmpFile, $"{newTrapName}"))
|
||||
return;
|
||||
}
|
||||
logger.LogInfo($"Identical trap file for {TrapFile} already exists");
|
||||
FileUtils.TryDelete(tmpFile);
|
||||
}
|
||||
}
|
||||
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
|
||||
{
|
||||
logger.LogError($"Failed to move the trap file from {tmpFile} to {TrapFile} because {ex}");
|
||||
}
|
||||
}
|
||||
|
||||
public void Emit(ITrapEmitter emitter)
|
||||
{
|
||||
emitter.EmitTrap(Writer);
|
||||
}
|
||||
|
||||
private static string TrapExtension(CompressionMode compression)
|
||||
{
|
||||
switch (compression)
|
||||
{
|
||||
case CompressionMode.None: return ".trap";
|
||||
case CompressionMode.Gzip: return ".trap.gz";
|
||||
case CompressionMode.Brotli: return ".trap.br";
|
||||
default: throw new ArgumentOutOfRangeException(nameof(compression), compression, "Unsupported compression type");
|
||||
}
|
||||
}
|
||||
|
||||
public static string TrapPath(ILogger logger, string? folder, PathTransformer.ITransformedPath path, TrapWriter.CompressionMode trapCompression)
|
||||
{
|
||||
var filename = $"{path.Value}{TrapExtension(trapCompression)}";
|
||||
if (string.IsNullOrEmpty(folder))
|
||||
folder = Directory.GetCurrentDirectory();
|
||||
|
||||
return FileUtils.NestPaths(logger, folder, filename);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user