From 43d7e598a56a5b80f6da7e30343679b7b751b53e Mon Sep 17 00:00:00 2001 From: Luke Cartey Date: Mon, 20 Aug 2018 12:46:50 +0100 Subject: [PATCH 01/58] C#: Treat GetFileName method call as sanitizer Use the GetFileName call as a sanitizer, rather than an argument to that call. It is the _result_ of the GetFileName call which should be considered sanitized. By using the argument, we can spuriously suppress use-use flow. Consider: ``` var path = Path.Combine(destDir, entry.GetFullName()); var fileName = Path.GetFileName(path); log("Extracting " + fileName); entry.ExtractToFile(path); ``` Previously, the `ExtractToFile(path)` call would not have been flagged, because the `path` argument to `GetFileName` was considered sanitized, and that argument formed a use-use pair with the `path` argument to `ExtractToFile`. Now, this result would be flagged because only the result of the `GetFileName` call is considered sanitized. --- csharp/ql/src/semmle/code/csharp/security/dataflow/ZipSlip.qll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/ql/src/semmle/code/csharp/security/dataflow/ZipSlip.qll b/csharp/ql/src/semmle/code/csharp/security/dataflow/ZipSlip.qll index cab63029688..cfa12872182 100644 --- a/csharp/ql/src/semmle/code/csharp/security/dataflow/ZipSlip.qll +++ b/csharp/ql/src/semmle/code/csharp/security/dataflow/ZipSlip.qll @@ -104,7 +104,7 @@ module ZipSlip { GetFileNameSanitizer() { exists(MethodCall mc | mc.getTarget().hasQualifiedName("System.IO.Path", "GetFileName") | - this.asExpr() = mc.getAnArgument() + this.asExpr() = mc ) } } From fc925d49f432308e315c9785c1f22ed9614f6329 Mon Sep 17 00:00:00 2001 From: Luke Cartey Date: Mon, 20 Aug 2018 12:52:36 +0100 Subject: [PATCH 02/58] C#: ZipSlip - Treat the result of Substring as sanitized. As with the previous commit, this considers the result of substring as sanitized, rather than the argument. --- .../code/csharp/security/dataflow/ZipSlip.qll | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/csharp/ql/src/semmle/code/csharp/security/dataflow/ZipSlip.qll b/csharp/ql/src/semmle/code/csharp/security/dataflow/ZipSlip.qll index cfa12872182..a2ec38e831c 100644 --- a/csharp/ql/src/semmle/code/csharp/security/dataflow/ZipSlip.qll +++ b/csharp/ql/src/semmle/code/csharp/security/dataflow/ZipSlip.qll @@ -96,7 +96,7 @@ module ZipSlip { } /** - * An argument to `GetFileName`. + * An call to `GetFileName`. * * This is considered a sanitizer because it extracts just the file name, not the full path. */ @@ -110,16 +110,30 @@ module ZipSlip { } /** - * A qualifier in a call to `StartsWith` or `Substring` string method. + * A call to Substring. * - * A call to a String method such as `StartsWith` or `Substring` can indicate a check for a + * This is considered a sanitizer because `Substring` may be used to extract a single component + * of a path to avoid ZipSlip. + */ + class SubstringSanitizer extends Sanitizer { + SubstringSanitizer() { + exists(MethodCall mc | + mc.getTarget().hasQualifiedName("System.String", "Substring") | + this.asExpr() = mc + ) + } + } + + /** + * A qualifier in a call to `StartsWith` string method. + * + * A call to a String method such as `StartsWith` can indicate a check for a * relative path, or a check against the destination folder for whitelisted/target path, etc. */ class StringCheckSanitizer extends Sanitizer { StringCheckSanitizer() { exists(MethodCall mc | - mc.getTarget().hasQualifiedName("System.String", "StartsWith") or - mc.getTarget().hasQualifiedName("System.String", "Substring") | + mc.getTarget().hasQualifiedName("System.String", "StartsWith") | this.asExpr() = mc.getQualifier() ) } From b1d5d5bf867d6499ffdd8f15efb4e299e6f92325 Mon Sep 17 00:00:00 2001 From: Luke Cartey Date: Mon, 20 Aug 2018 12:55:38 +0100 Subject: [PATCH 03/58] C#: ZipSlip - Refine `StartsWith` sanitizer. ZipSlip can be avoided by checking that the combined and resolved path `StartsWith` the appropriate destination directory. Refine the `StartsWith` sanitizer to: * Consider expressions guarded by an appropriate StartsWith check to be sanitized. * Consider a StartsWith check to be inappropriate if it is checking the result of `Path.Combine`, as that has not been appropriately resolved. Tests have been updated to reflect this refinement. --- .../code/csharp/security/dataflow/ZipSlip.qll | 20 +++++++++++++++---- .../CWE-022/ZipSlip/ZipSlip.cs | 12 +++++++++-- .../CWE-022/ZipSlip/ZipSlip.expected | 10 ++++++---- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/csharp/ql/src/semmle/code/csharp/security/dataflow/ZipSlip.qll b/csharp/ql/src/semmle/code/csharp/security/dataflow/ZipSlip.qll index a2ec38e831c..cd79950feac 100644 --- a/csharp/ql/src/semmle/code/csharp/security/dataflow/ZipSlip.qll +++ b/csharp/ql/src/semmle/code/csharp/security/dataflow/ZipSlip.qll @@ -4,6 +4,8 @@ import csharp module ZipSlip { + import semmle.code.csharp.controlflow.Guards + /** * A data flow source for unsafe zip extraction. */ @@ -125,16 +127,26 @@ module ZipSlip { } /** - * A qualifier in a call to `StartsWith` string method. + * An expression which is guarded by a call to `StartsWith`. * * A call to a String method such as `StartsWith` can indicate a check for a * relative path, or a check against the destination folder for whitelisted/target path, etc. */ class StringCheckSanitizer extends Sanitizer { StringCheckSanitizer() { - exists(MethodCall mc | - mc.getTarget().hasQualifiedName("System.String", "StartsWith") | - this.asExpr() = mc.getQualifier() + exists(GuardedExpr ge, MethodCall mc, Expr startsWithQualifier | + ge = this.asExpr() and + ge.isGuardedBy(mc, startsWithQualifier, true) | + mc.getTarget().hasQualifiedName("System.String", "StartsWith") and + mc.getQualifier() = startsWithQualifier and + /* + * A StartsWith check against Path.Combine is not sufficient, because the ".." elements have + * not yet been resolved. + */ + not exists(MethodCall combineCall | + combineCall.getTarget().hasQualifiedName("System.IO.Path", "Combine") and + DataFlow::localFlow(DataFlow::exprNode(combineCall), DataFlow::exprNode(startsWithQualifier)) + ) ) } } diff --git a/csharp/ql/test/query-tests/Security Features/CWE-022/ZipSlip/ZipSlip.cs b/csharp/ql/test/query-tests/Security Features/CWE-022/ZipSlip/ZipSlip.cs index 2206333d414..d82f76ced0a 100644 --- a/csharp/ql/test/query-tests/Security Features/CWE-022/ZipSlip/ZipSlip.cs +++ b/csharp/ql/test/query-tests/Security Features/CWE-022/ZipSlip/ZipSlip.cs @@ -31,7 +31,15 @@ namespace ZipSlip string destFilePath = Path.Combine(destDirectory, fullPath); entry.ExtractToFile(destFilePath, true); - // GOOD: some check on destination. + // BAD: destFilePath isn't fully resolved, so may still contain .. + if (destFilePath.StartsWith(destDirectory)) + entry.ExtractToFile(destFilePath, true); + + // BAD + destFilePath = Path.GetFullPath(Path.Combine(destDirectory, fullPath)); + entry.ExtractToFile(destFilePath, true); + + // GOOD: a check for StartsWith against a fully resolved path if (destFilePath.StartsWith(destDirectory)) entry.ExtractToFile(destFilePath, true); } @@ -115,7 +123,7 @@ namespace ZipSlip // GOOD: the path is checked in this extension method archive.ExtractToDirectory(targetPath); - UnzipToStream(file, targetPath); + UnzipToStream(file, targetPath); } } } diff --git a/csharp/ql/test/query-tests/Security Features/CWE-022/ZipSlip/ZipSlip.expected b/csharp/ql/test/query-tests/Security Features/CWE-022/ZipSlip/ZipSlip.expected index c4219129d01..f9da54b1b80 100644 --- a/csharp/ql/test/query-tests/Security Features/CWE-022/ZipSlip/ZipSlip.expected +++ b/csharp/ql/test/query-tests/Security Features/CWE-022/ZipSlip/ZipSlip.expected @@ -1,7 +1,9 @@ | ZipSlip.cs:24:41:24:52 | access to local variable destFileName | Unsanitized zip archive $@, which may contain '..', is used in a file system operation. | ZipSlip.cs:19:31:19:44 | access to property FullName | item path | | ZipSlip.cs:32:41:32:52 | access to local variable destFilePath | Unsanitized zip archive $@, which may contain '..', is used in a file system operation. | ZipSlip.cs:16:52:16:65 | access to property FullName | item path | -| ZipSlip.cs:61:74:61:85 | access to local variable destFilePath | Unsanitized zip archive $@, which may contain '..', is used in a file system operation. | ZipSlip.cs:54:72:54:85 | access to property FullName | item path | -| ZipSlip.cs:68:71:68:82 | access to local variable destFilePath | Unsanitized zip archive $@, which may contain '..', is used in a file system operation. | ZipSlip.cs:54:72:54:85 | access to property FullName | item path | -| ZipSlip.cs:75:57:75:68 | access to local variable destFilePath | Unsanitized zip archive $@, which may contain '..', is used in a file system operation. | ZipSlip.cs:54:72:54:85 | access to property FullName | item path | -| ZipSlip.cs:83:58:83:69 | access to local variable destFilePath | Unsanitized zip archive $@, which may contain '..', is used in a file system operation. | ZipSlip.cs:54:72:54:85 | access to property FullName | item path | +| ZipSlip.cs:36:45:36:56 | access to local variable destFilePath | Unsanitized zip archive $@, which may contain '..', is used in a file system operation. | ZipSlip.cs:16:52:16:65 | access to property FullName | item path | +| ZipSlip.cs:40:41:40:52 | access to local variable destFilePath | Unsanitized zip archive $@, which may contain '..', is used in a file system operation. | ZipSlip.cs:16:52:16:65 | access to property FullName | item path | +| ZipSlip.cs:69:74:69:85 | access to local variable destFilePath | Unsanitized zip archive $@, which may contain '..', is used in a file system operation. | ZipSlip.cs:62:72:62:85 | access to property FullName | item path | +| ZipSlip.cs:76:71:76:82 | access to local variable destFilePath | Unsanitized zip archive $@, which may contain '..', is used in a file system operation. | ZipSlip.cs:62:72:62:85 | access to property FullName | item path | +| ZipSlip.cs:83:57:83:68 | access to local variable destFilePath | Unsanitized zip archive $@, which may contain '..', is used in a file system operation. | ZipSlip.cs:62:72:62:85 | access to property FullName | item path | +| ZipSlip.cs:91:58:91:69 | access to local variable destFilePath | Unsanitized zip archive $@, which may contain '..', is used in a file system operation. | ZipSlip.cs:62:72:62:85 | access to property FullName | item path | | ZipSlipBad.cs:10:29:10:40 | access to local variable destFileName | Unsanitized zip archive $@, which may contain '..', is used in a file system operation. | ZipSlipBad.cs:9:59:9:72 | access to property FullName | item path | From f9227eeee543a09b9f551225f7822606c8e9b520 Mon Sep 17 00:00:00 2001 From: Luke Cartey Date: Fri, 24 Aug 2018 14:29:35 +0100 Subject: [PATCH 04/58] C#: ZipSlip - Module documentation improvements. --- .../src/semmle/code/csharp/security/dataflow/ZipSlip.qll | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/csharp/ql/src/semmle/code/csharp/security/dataflow/ZipSlip.qll b/csharp/ql/src/semmle/code/csharp/security/dataflow/ZipSlip.qll index cd79950feac..a58d131e0f7 100644 --- a/csharp/ql/src/semmle/code/csharp/security/dataflow/ZipSlip.qll +++ b/csharp/ql/src/semmle/code/csharp/security/dataflow/ZipSlip.qll @@ -98,7 +98,7 @@ module ZipSlip { } /** - * An call to `GetFileName`. + * A call to `GetFileName`. * * This is considered a sanitizer because it extracts just the file name, not the full path. */ @@ -127,10 +127,10 @@ module ZipSlip { } /** - * An expression which is guarded by a call to `StartsWith`. + * An expression which is guarded by a call to `String.StartsWith`. * - * A call to a String method such as `StartsWith` can indicate a check for a - * relative path, or a check against the destination folder for whitelisted/target path, etc. + * A call to the method `String.StartsWith` can indicate the the tainted path value is being + * validated to ensure that it occurs within a permitted output path. */ class StringCheckSanitizer extends Sanitizer { StringCheckSanitizer() { From 593f0a9d71a5bcd2e0b0350c171f15053cb00c47 Mon Sep 17 00:00:00 2001 From: calum Date: Wed, 19 Sep 2018 12:02:59 +0100 Subject: [PATCH 05/58] C#: Implement query and script for generating C# qltest stubs. --- csharp/ql/src/Stubs/MinimalStubsFromSource.ql | 31 ++ csharp/ql/src/Stubs/Stubs.qll | 448 ++++++++++++++++++ csharp/ql/src/Stubs/make_stubs.py | 114 +++++ .../Stubs/MinimalStubsFromSource.expected | 1 + .../Stubs/MinimalStubsFromSource.qlref | 1 + csharp/ql/test/query-tests/Stubs/Test.cs | 125 +++++ 6 files changed, 720 insertions(+) create mode 100644 csharp/ql/src/Stubs/MinimalStubsFromSource.ql create mode 100644 csharp/ql/src/Stubs/Stubs.qll create mode 100644 csharp/ql/src/Stubs/make_stubs.py create mode 100644 csharp/ql/test/query-tests/Stubs/MinimalStubsFromSource.expected create mode 100644 csharp/ql/test/query-tests/Stubs/MinimalStubsFromSource.qlref create mode 100644 csharp/ql/test/query-tests/Stubs/Test.cs diff --git a/csharp/ql/src/Stubs/MinimalStubsFromSource.ql b/csharp/ql/src/Stubs/MinimalStubsFromSource.ql new file mode 100644 index 00000000000..e8751ded952 --- /dev/null +++ b/csharp/ql/src/Stubs/MinimalStubsFromSource.ql @@ -0,0 +1,31 @@ +/* + * Tool to generate C# stubs from a qltest snapshot. + * + * It finds all declarations used in the source code, + * and generates minimal C# stubs containing those declarations + * and their dependencies. + */ + +import csharp +import Stubs + +/** Declarations used by source code. */ +class UsedInSource extends GeneratedDeclaration { + UsedInSource() { + ( + this = any(Access a).getTarget() + or + this = any(Call c).getTarget() + or + exists(ValueOrRefType t | t.fromSource() | this = t.getABaseType()) + or + exists(Variable v | v.fromSource() | this = v.getType()) + or + exists(Virtualizable v | v.fromSource() | this = v.getImplementee() or this = v.getOverridee()) + ) + and + this.fromLibrary() + } +} + +select generatedCode() diff --git a/csharp/ql/src/Stubs/Stubs.qll b/csharp/ql/src/Stubs/Stubs.qll new file mode 100644 index 00000000000..c7a971e166f --- /dev/null +++ b/csharp/ql/src/Stubs/Stubs.qll @@ -0,0 +1,448 @@ +/** + * Generates C# stubs for use in test code. + * + * Extend the abstract class `GeneratedDeclaration` with the declarations that should be generated. + * This will generate stubs for all the required dependencies as well. + * + * Use + * ``` + * select generatedCode() + * ``` + * to retrieve the generated C# code. + */ + +import csharp + +/** An element that should be in the generated code. */ +abstract class GeneratedElement extends Element { +} + +/** A member that should be in the generated code. */ +abstract class GeneratedMember extends Member, GeneratedElement { +} + +/** A type that should be in the generated code. */ +private abstract class GeneratedType extends ValueOrRefType, GeneratedElement { + + GeneratedType() { + ( + this instanceof Interface + or + this instanceof Class + or + this instanceof Struct + or + this instanceof Enum + or + this instanceof DelegateType + ) + and + not this instanceof ConstructedType + and + not this.getALocation() instanceof ExcludedAssembly + and + this.fromLibrary() + } + + /** + * Holds if this type is duplicated in another assembly. + * In this case, we use the assembly with the highest string. + */ + private predicate isDuplicate() { + exists(GeneratedType dup | + dup.getQualifiedName() = this.getQualifiedName() and + this.getLocation().(Assembly).toString() < dup.getLocation().(Assembly).toString() + ) + } + + private string stubKeyword() { + this instanceof Interface and result = "interface" + or + this instanceof Struct and result = "struct" + or + this instanceof Class and result = "class" + or + this instanceof Enum and result = "enum" + } + + private string stubAbstractModifier() { + if this.(Class).isAbstract() then + result = "abstract " + else + result = "" + } + + private string stubStaticModifier() { + if this.isStatic() then result = "static " + else result = "" + } + + private string stubAttributes() { + if this.getAnAttribute().getType().getQualifiedName() = "System.FlagsAttribute" then + result = "[System.Flags]\n" + else + result = "" + } + + private string stubComment() { + result = "// Generated from `" + this.getQualifiedName() + "` in `" + + min(this.getLocation().toString()) + "`\n" + } + + private string stubAccessibilityModifier() { + if this.isPublic() then + result = "public " + else + result = "" + } + + /** Gets the entire C# stub code for this type. */ + final string getStub() { + if this.isDuplicate() then + result = "" + else + result = this.stubComment() + + this.stubAttributes() + + this.stubAbstractModifier() + + this.stubStaticModifier() + + this.stubAccessibilityModifier() + + this.stubKeyword() + " " + + this.getUndecoratedName() + + stubGenericArguments(this) + + stubBaseTypesString() + + "\n{\n" + + stubMembers() + + "}\n\n" + } + + private ValueOrRefType getAnInterestingBaseType() { + result = this.getABaseType() and + not result instanceof ObjectType + } + + private string stubBaseTypesString() { + if this instanceof Enum then + result = "" + else if exists(getAnInterestingBaseType()) then + result = " : " + + concat(int i, ValueOrRefType t | + t = this.getAnInterestingBaseType() and (if t instanceof Class then i=0 else i=1) | + stubClassName(t), ", " order by i + ) + else + result = "" + } + + private string stubMembers() { + result = concat(stubMember(this.getAGeneratedMember())) + } + + private GeneratedMember getAGeneratedMember() { + result.getDeclaringType() = this + } + + final Type getAGeneratedType() { + result = getAnInterestingBaseType() + or + result = getAGeneratedMember().(Callable).getReturnType() + or + result = getAGeneratedMember().(Callable).getAParameter().getType() + or + result = getAGeneratedMember().(Property).getType() + or + result = getAGeneratedMember().(Field).getType() + } +} + +/** + * A declaration that should be generated. + * This is extended in client code to identify the actual + * declarations that should be generated. + */ +abstract class GeneratedDeclaration extends Declaration { +} + +private class IndirectType extends GeneratedType { + + IndirectType() { + this.getASubType() instanceof GeneratedType + or + this.getAChildType() instanceof GeneratedType + or + this.(UnboundGenericType).getAConstructedGeneric().getASubType() instanceof GeneratedType + or + exists(GeneratedType t | this = getAContainedType(t.getAGeneratedType())) + or + exists(GeneratedDeclaration decl | decl.(Member).getDeclaringType() = this) + } +} + +private class RootGeneratedType extends GeneratedType { + RootGeneratedType() { + this instanceof GeneratedDeclaration + } +} + +private Type getAContainedType(Type t) { + result = t + or + result = getAContainedType(t.(ConstructedType).getATypeArgument()) +} + +private class RootGeneratedMember extends GeneratedMember { + RootGeneratedMember() { + this instanceof GeneratedDeclaration + } +} + +private predicate declarationExists(Virtualizable m) { + m instanceof GeneratedMember + or + m.getLocation() instanceof ExcludedAssembly +} + +private class InheritedMember extends GeneratedMember, Virtualizable { + InheritedMember() { + declarationExists(this.getImplementee+()) + or + declarationExists(this.getAnImplementor+()) + or + declarationExists(this.getOverridee+()) + or + declarationExists(this.getAnOverrider*()) + } +} + +/** A namespace that contains at least one generated type. */ +private class GeneratedNamespace extends Namespace, GeneratedElement { + GeneratedNamespace() { + this.getATypeDeclaration() instanceof GeneratedType + or + this.getAChildNamespace() instanceof GeneratedNamespace + } + + private string getPreamble() { + if this.isGlobalNamespace() then + result = "" + else + result = "namespace " + this.getName() + "\n{\n" + } + + private string getPostAmble() { + if this.isGlobalNamespace() then + result = "" + else + result = "}\n" + } + + final string getStubs() { + result = getPreamble() + + getTypeStubs() + + getSubNamespaces(0) + + getPostAmble() + } + + /** Gets the `n`th generated child namespace, indexed from 0. */ + pragma[nomagic] + final GeneratedNamespace getChildNamespace(int n) { + result.getParentNamespace() = this and + result.getName() = rank[n+1](GeneratedNamespace g | g.getParentNamespace() = this | g.getName()) + } + + final int getChildNamespaceCount() { + result = count(GeneratedNamespace g | g.getParentNamespace() = this) + } + + private string getSubNamespaces(int n) { + result = getChildNamespace(n).getStubs() + getSubNamespaces(n+1) + or + n = getChildNamespaceCount() and result = "" + } + + private string getTypeStubs() { + result = concat(string s | s = any(GeneratedType gt | gt.getDeclaringNamespace() = this).getStub()) + } +} + +/** + * Specify assemblies to exclude. + * Do not generate any types from these assemblies. + */ +abstract class ExcludedAssembly extends Assembly { +} + +/** Exclude types from these standard assemblies. */ +private class DefaultLibs extends ExcludedAssembly { + DefaultLibs() { + this.getName() = "System.Private.CoreLib" + or this.getName() = "mscorlib" + or this.getName() = "System.Runtime" + } +} + +private string stubAccessibility(Member m) { + if m.getDeclaringType() instanceof Interface or exists(m.(Virtualizable).getExplicitlyImplementedInterface()) then result = "" + else if m.isPublic() then result = "public " + else if m.isProtected() then result = "protected " + else if m.isPrivate() then result = "private " + else if m.isInternal() then result = "internal " + else result = "unknown-accessibility" +} + +private string stubModifiers(Member m) { + result = stubAccessibility(m) + stubStatic(m) + stubOverride(m) +} + +private string stubStatic(Member m) { + if m.(Modifiable).isStatic() then result = "static " + else result = "" +} + +private string stubOverride(Member m) { + if m.getDeclaringType() instanceof Interface then result = "" + else if m.(Virtualizable).isVirtual() then result = "virtual " + else if m.(Virtualizable).isAbstract() then result = "abstract " + else if m.(Virtualizable).isOverride() then result = "override " + else result = "" +} + +private string stubQualifiedNamePrefix(ValueOrRefType t) { + if t.getParent() instanceof GlobalNamespace then result="" + else if t.getParent() instanceof Namespace then result = t.getParent().(Namespace).getQualifiedName() + "." + else result = stubQualifiedNamePrefix(t.getParent()) + "." +} + +language[monotonicAggregates] +private string stubClassName(Type t) { + if t instanceof ObjectType then + result = "object" + else if t instanceof StringType then + result = "string" + else if t instanceof IntType then + result = "int" + else if t instanceof BoolType then + result = "bool" + else if t instanceof VoidType then + result = "void" + else if t instanceof FloatType then + result = "float" + else if t instanceof DoubleType then + result = "double" + else if t instanceof TypeParameter then + result = t.getName() + else if t instanceof ArrayType then + result = stubClassName(t.(ArrayType).getElementType()) + "[]" + else if t instanceof PointerType then + result = stubClassName(t.(PointerType).getReferentType()) + "*" + else if t instanceof TupleType then + result = "(" + concat(int i, Type element | element = t.(TupleType).getElementType(i) | stubClassName(element), "," order by i) + ")" + else if t instanceof ValueOrRefType then + result = stubQualifiedNamePrefix(t) + t.getUndecoratedName() + stubGenericArguments(t) + else + result = "" +} + +language[monotonicAggregates] +private string stubGenericArguments(ValueOrRefType t) { + if t instanceof UnboundGenericType then + result = "<" + concat(int n | exists(t.(UnboundGenericType).getTypeParameter(n)) | t.(UnboundGenericType).getTypeParameter(n).getName(),",") + ">" + else if t instanceof ConstructedType then + result = "<" + concat(int n | exists(t.(ConstructedType).getTypeArgument(n)) | stubClassName(t.(ConstructedType).getTypeArgument(n)),",") + ">" + else + result = "" +} + +private string stubGenericMethodParams(Method m) { + if m instanceof UnboundGenericMethod then + result = "<" + concat(int n, TypeParameter param | param = m.(UnboundGenericMethod).getTypeParameter(n) | param.getName(), "," order by n) + ">" + else + result = "" +} + +private string stubImplementation(Virtualizable c) { + if c.isAbstract() or c.getDeclaringType() instanceof Interface + then result = "" + else result = " => throw null" +} + +private string stubParameters(Parameterizable p) { + result = concat(int i, Parameter param | param = p.getParameter(i) and not param.getType() instanceof ArglistType | + stubParameterModifiers(param) + stubClassName(param.getType()) + " " + param.getName() + stubDefaultValue(param), ", " order by i) +} + +private string stubParameterModifiers(Parameter p) { + if p.isOut() then result = "out " + else if p.isRef() then result = "ref " + else if p.isParams() then result = "params " + else if p.isIn() then result = "" // Only C# 7.1 so ignore + else if p.hasExtensionMethodModifier() then result = "this " + else result = "" +} + +private string stubDefaultValue(Parameter p) { + if p.hasDefaultValue() + then result = " = default(" + stubClassName(p.getType()) + ")" + else result = "" +} + +private string stubExplicitImplementation(Member c) { + if exists(c.(Virtualizable).getExplicitlyImplementedInterface()) + then result = stubClassName(c.(Virtualizable).getExplicitlyImplementedInterface()) + "." + else result = "" +} + +private string stubMember(Member m) { + exists(Method c | m = c and not m.getDeclaringType() instanceof Enum | + result = " " + stubModifiers(c) + stubClassName(c.getReturnType()) + " " + + stubExplicitImplementation(c) + c.getName() + stubGenericMethodParams(c) + + "(" + stubParameters(c) + ")" + stubImplementation(c) + ";\n" + ) + or + exists(Operator op | m = op and not m.getDeclaringType() instanceof Enum | + result = " " + stubModifiers(op) + stubClassName(op.getReturnType()) + " operator " + + op.getName() +"(" + stubParameters(op) + ") => throw null;\n" + ) + or + result = " " + m.(EnumConstant).getName() + ",\n" + or + exists(Property p | m = p | + result = " " + stubModifiers(m) + stubClassName(p.getType()) + " " + + stubExplicitImplementation(p) + p.getName() + " { " + stubGetter(p) + stubSetter(p) + "}\n" + ) + or exists(Constructor c | m = c and not c.getDeclaringType() instanceof Enum | + result = " " + stubModifiers(m) + c.getName() + "(" + stubParameters(c) + + ") => throw null;\n" + ) + or exists(Indexer i | m = i | + result = " " + stubModifiers(m) + stubClassName(i.getType()) + " this[" + stubParameters(i) + + "] { " + stubGetter(i) + stubSetter(i) + "}\n" + ) + or exists(Field f | f = m and not f instanceof EnumConstant | + result = " " + stubModifiers(m) + stubClassName(f.getType()) + " " + f.getName() + ";\n" + ) +} + +private string stubGetter(DeclarationWithGetSetAccessors p) { + if exists(p.getGetter()) then ( + if p.isAbstract() or p.getDeclaringType() instanceof Interface then + result = "get; " + else + result = "get => throw null; " + ) else + result = "" +} + +private string stubSetter(DeclarationWithGetSetAccessors p) { + if exists(p.getSetter()) then ( + if p.isAbstract() or p.getDeclaringType() instanceof Interface then + result = "set; " + else + result = "set => throw null; " + ) else + result = "" +} + +/** Gets the generated C# code. */ +string generatedCode() { + result = any(GeneratedNamespace ns | ns.isGlobalNamespace()).getStubs() +} diff --git a/csharp/ql/src/Stubs/make_stubs.py b/csharp/ql/src/Stubs/make_stubs.py new file mode 100644 index 00000000000..980da6da19f --- /dev/null +++ b/csharp/ql/src/Stubs/make_stubs.py @@ -0,0 +1,114 @@ +# Tool to generate 'stub.cs' files inside C# qltest projects. +# The purpose of this is to stub out assemblies from the qltest case +# so that the test is self-contained and will still run without them. +# +# To do this, it +# 1. Performs a regular qltest to generate a snapshot +# 2. Runs a QL query on the snapshot to find out which symbols are needed, +# then uses QL to generate a string of C# code. +# 3. Re-runs the test to ensure that it still compiles and passes. + +import sys +import os + +print('Script to generate stub.cs files for C# qltest projects') + +if len(sys.argv) < 2: + print("Please supply a qltest directory.") + exit(1) + +testDir = sys.argv[1] + +if not os.path.isdir(testDir): + print("Directory", testDir, "does not exist") + exit(1) + +# Does it contain a .ql file and a .cs file? + +foundCS = False +foundQL = False + +for file in os.listdir(testDir): + if file.endswith(".cs"): + foundCS = True + if file.endswith(".ql") or file.endswith(".qlref"): + foundQL = True + +if not foundQL: + print("Test directory does not contain .ql files. Please specify a working qltest directory.") + exit(1) + +if not foundCS: + print("Test directory does not contain .cs files. Please specify a working qltest directory.") + exit(1) + +cmd = 'odasa selfTest' +print(cmd) +if os.system(cmd): + print("odasa selfTest failed. Ensure odasa is on your current path.") + exit(1) + +csharpQueries = os.path.abspath(os.path.dirname(sys.argv[0])) +outputFile = os.path.join(testDir, 'stubs.cs') + +print("Stubbing qltest in", testDir) + +if os.path.isfile(outputFile): + os.remove(outputFile) # It would interfere with the test. + print("Removed previous", outputFile) + +cmd = 'odasa qltest --optimize --leave-temp-files "' + testDir + '"' +print(cmd) +if os.system(cmd) != 0: + print("qltest failed. Please fix up the test before proceeding.") + exit(1) + +dbDir = os.path.join(testDir, os.path.basename(testDir) + ".testproj", "db-csharp") + +if not os.path.isdir(dbDir): + print("Expected database directory " + dbDir + " not found. Please contact Semmle.") + exit(1) + +cmd = 'odasa runQuery --query "' + os.path.join(csharpQueries, 'MinimalStubsFromSource.ql') + '" --db "' + dbDir +'" --output-file "' + outputFile + '"' +print(cmd) +if os.system(cmd): + print('Failed to run the query to generate output file. Please contact Semmle.') + exit(1) + +# Remove the leading " and trailing " bytes from the file +len = os.stat(outputFile).st_size +f = open(outputFile, "rb") +try: + quote = f.read(1) + if quote != b'"': + print("Unexpected character in file. Please contact Semmle.") + contents = f.read(len-3) + quote = f.read(1) + if quote != b'"': + print("Unexpected end character. Please contact Semmle.", quote) +finally: + f.close() + +f = open(outputFile, "wb") +f.write(contents) +f.close() + +cmd = 'odasa qltest --optimize "' + testDir + '"' +print(cmd) + +if os.system(cmd): + print('\nTest failed. You may need to fix up', outputFile) + print('It may help to view', outputFile, ' in Visual Studio') + print("Next steps:") + print('1. Look at the compilation errors, and fix up', outputFile, 'so that the test compiles') + print('2. Re-run odasa qltest --optimize "' + testDir + '"') + print('3. git add "' + outputFile + '"') + exit(1) + +print("\nStub generation successful! Next steps:") +print('1. Edit "semmle-extractor-options" in the .cs files to remove unused references') +print('2. Re-run odasa qltest --optimize "' + testDir + '"') +print('3. git add "' + outputFile + '"') +print('4. Commit your changes.') + +exit(0) diff --git a/csharp/ql/test/query-tests/Stubs/MinimalStubsFromSource.expected b/csharp/ql/test/query-tests/Stubs/MinimalStubsFromSource.expected new file mode 100644 index 00000000000..f1b5fd84ab7 --- /dev/null +++ b/csharp/ql/test/query-tests/Stubs/MinimalStubsFromSource.expected @@ -0,0 +1 @@ +| namespace System\n{\n// Generated from `System.Uri` in `System.Private.Uri, Version=4.0.4.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class Uri : System.Runtime.Serialization.ISerializable\n{\n public override bool Equals(object comparand) => throw null;\n public override int GetHashCode() => throw null;\n public override string ToString() => throw null;\n void System.Runtime.Serialization.ISerializable.GetObjectData(System.Runtime.Serialization.SerializationInfo serializationInfo, System.Runtime.Serialization.StreamingContext streamingContext) => throw null;\n}\n\nnamespace Collections\n{\n// Generated from `System.Collections.ArrayList` in `System.Runtime.Extensions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class ArrayList : System.ICloneable, System.Collections.IList, System.Collections.IEnumerable, System.Collections.ICollection\n{\n public static System.Collections.ArrayList FixedSize(System.Collections.ArrayList list) => throw null;\n public virtual System.Collections.IEnumerator GetEnumerator() => throw null;\n public virtual bool Contains(object item) => throw null;\n public virtual bool IsFixedSize { get => throw null; }\n public virtual bool IsReadOnly { get => throw null; }\n public virtual bool IsSynchronized { get => throw null; }\n public virtual int Add(object value) => throw null;\n public virtual int Count { get => throw null; }\n public virtual int IndexOf(object value) => throw null;\n public virtual object Clone() => throw null;\n public virtual object SyncRoot { get => throw null; }\n public virtual object this[int index] { get => throw null; set => throw null; }\n public virtual void Clear() => throw null;\n public virtual void CopyTo(System.Array array, int arrayIndex) => throw null;\n public virtual void Insert(int index, object value) => throw null;\n public virtual void Remove(object obj) => throw null;\n public virtual void RemoveAt(int index) => throw null;\n}\n\n// Generated from `System.Collections.SortedList` in `System.Collections.NonGeneric, Version=4.1.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class SortedList : System.ICloneable, System.Collections.IEnumerable, System.Collections.IDictionary, System.Collections.ICollection\n{\n System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => throw null;\n public virtual System.Collections.ICollection Keys { get => throw null; }\n public virtual System.Collections.ICollection Values { get => throw null; }\n public virtual System.Collections.IDictionaryEnumerator GetEnumerator() => throw null;\n public virtual bool Contains(object key) => throw null;\n public virtual bool IsFixedSize { get => throw null; }\n public virtual bool IsReadOnly { get => throw null; }\n public virtual bool IsSynchronized { get => throw null; }\n public virtual int Count { get => throw null; }\n public virtual object Clone() => throw null;\n public virtual object GetByIndex(int index) => throw null;\n public virtual object SyncRoot { get => throw null; }\n public virtual object this[object key] { get => throw null; set => throw null; }\n public virtual void Add(object key, object value) => throw null;\n public virtual void Clear() => throw null;\n public virtual void CopyTo(System.Array array, int arrayIndex) => throw null;\n public virtual void Remove(object key) => throw null;\n}\n\nnamespace Specialized\n{\n// Generated from `System.Collections.Specialized.NameObjectCollectionBase` in `System.Collections.Specialized, Version=4.1.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\nabstract public class NameObjectCollectionBase : System.Runtime.Serialization.ISerializable, System.Runtime.Serialization.IDeserializationCallback, System.Collections.IEnumerable, System.Collections.ICollection\n{\n bool System.Collections.ICollection.IsSynchronized { get => throw null; }\n object System.Collections.ICollection.SyncRoot { get => throw null; }\n public virtual System.Collections.IEnumerator GetEnumerator() => throw null;\n public virtual int Count { get => throw null; }\n public virtual void GetObjectData(System.Runtime.Serialization.SerializationInfo info, System.Runtime.Serialization.StreamingContext context) => throw null;\n public virtual void OnDeserialization(object sender) => throw null;\n void System.Collections.ICollection.CopyTo(System.Array array, int index) => throw null;\n}\n\n// Generated from `System.Collections.Specialized.NameValueCollection` in `System.Collections.Specialized, Version=4.1.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class NameValueCollection : System.Collections.Specialized.NameObjectCollectionBase\n{\n public string this[string name] { get => throw null; set => throw null; }\n}\n\n}\n}\nnamespace IO\n{\n// Generated from `System.IO.StringReader` in `System.Runtime.Extensions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class StringReader : System.IO.TextReader\n{\n public StringReader(string s) => throw null;\n}\n\n// Generated from `System.IO.TextReader` in `System.Runtime.Extensions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\nabstract public class TextReader : System.MarshalByRefObject, System.IDisposable\n{\n public void Dispose() => throw null;\n}\n\n}\nnamespace Linq\n{\n// Generated from `System.Linq.Enumerable` in `System.Linq, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\nstatic public class Enumerable\n{\n public static System.Collections.Generic.IEnumerable Select(this System.Collections.Generic.IEnumerable source, System.Func selector) => throw null;\n}\n\n// Generated from `System.Linq.IQueryable` in `System.Linq.Expressions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic interface IQueryable : System.Collections.IEnumerable\n{\n}\n\n// Generated from `System.Linq.ParallelEnumerable` in `System.Linq.Parallel, Version=4.0.3.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\nstatic public class ParallelEnumerable\n{\n public static System.Linq.ParallelQuery AsParallel(this System.Collections.IEnumerable source) => throw null;\n}\n\n// Generated from `System.Linq.ParallelQuery` in `System.Linq.Parallel, Version=4.0.3.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class ParallelQuery : System.Collections.IEnumerable\n{\n System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => throw null;\n}\n\n// Generated from `System.Linq.Queryable` in `System.Linq.Queryable, Version=4.0.3.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\nstatic public class Queryable\n{\n public static System.Linq.IQueryable AsQueryable(this System.Collections.IEnumerable source) => throw null;\n}\n\n}\nnamespace Text\n{\nnamespace RegularExpressions\n{\n// Generated from `System.Text.RegularExpressions.Capture` in `System.Text.RegularExpressions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class Capture\n{\n public override string ToString() => throw null;\n}\n\n// Generated from `System.Text.RegularExpressions.Group` in `System.Text.RegularExpressions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class Group : System.Text.RegularExpressions.Capture\n{\n}\n\n// Generated from `System.Text.RegularExpressions.Match` in `System.Text.RegularExpressions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class Match : System.Text.RegularExpressions.Group\n{\n}\n\n// Generated from `System.Text.RegularExpressions.RegexOptions` in `System.Text.RegularExpressions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic enum RegexOptions\n{\n IgnoreCase,\n}\n\n// Generated from `System.Text.RegularExpressions.Regex` in `System.Text.RegularExpressions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class Regex : System.Runtime.Serialization.ISerializable\n{\n public Regex(string pattern) => throw null;\n public Regex(string pattern, System.Text.RegularExpressions.RegexOptions options, System.TimeSpan matchTimeout) => throw null;\n public System.Text.RegularExpressions.Match Match(string input) => throw null;\n public override string ToString() => throw null;\n public static System.Text.RegularExpressions.Match Match(string input, string pattern) => throw null;\n public static System.Text.RegularExpressions.Match Match(string input, string pattern, System.Text.RegularExpressions.RegexOptions options, System.TimeSpan matchTimeout) => throw null;\n public string Replace(string input, string replacement) => throw null;\n void System.Runtime.Serialization.ISerializable.GetObjectData(System.Runtime.Serialization.SerializationInfo si, System.Runtime.Serialization.StreamingContext context) => throw null;\n}\n\n}\n}\n}\n | diff --git a/csharp/ql/test/query-tests/Stubs/MinimalStubsFromSource.qlref b/csharp/ql/test/query-tests/Stubs/MinimalStubsFromSource.qlref new file mode 100644 index 00000000000..53cbcd647b7 --- /dev/null +++ b/csharp/ql/test/query-tests/Stubs/MinimalStubsFromSource.qlref @@ -0,0 +1 @@ +Stubs/MinimalStubsFromSource.ql diff --git a/csharp/ql/test/query-tests/Stubs/Test.cs b/csharp/ql/test/query-tests/Stubs/Test.cs new file mode 100644 index 00000000000..f0f0e6f82dd --- /dev/null +++ b/csharp/ql/test/query-tests/Stubs/Test.cs @@ -0,0 +1,125 @@ +// semmle-extractor-options: /r:System.Text.RegularExpressions.dll /r:System.Collections.Specialized.dll /r:System.Net.dll /r:System.Web.dll /r:System.Net.HttpListener.dll /r:System.Collections.Specialized.dll /r:System.Private.Uri.dll /r:System.Runtime.Extensions.dll /r:System.Linq.Parallel.dll /r:System.Collections.Concurrent.dll /r:System.Linq.Expressions.dll /r:System.Collections.dll /r:System.Linq.Queryable.dll /r:System.Linq.dll /r:System.Collections.NonGeneric.dll /r:System.ObjectModel.dll /r:System.ComponentModel.TypeConverter.dll /r:System.IO.Compression.dll /r:System.IO.Pipes.dll /r:System.Net.Primitives.dll /r:System.Net.Security.dll /r:System.Security.Cryptography.Primitives.dll /r:System.Text.RegularExpressions.dll ${testdir}/../../resources/stubs/System.Web.cs /r:System.Runtime.Serialization.Primitives.dll + +using System; +using System.IO; +using System.Text; +using System.Collections; +using System.Collections.Generic; +using System.Collections.Specialized; +using System.Linq; +using System.Runtime.Serialization; +using System.Threading.Tasks; +using System.Web; +using System.Web.UI.WebControls; +using System.Text.RegularExpressions; + +public class RegexHandler +{ + private static readonly string JAVA_CLASS_REGEX = "^(([a-z])+.)+[A-Z]([a-z])+$"; + + public void ProcessRequest() + { + string userInput = ""; + + // BAD: + // Artificial regexes + new Regex("^([a-z]+)+$").Match(userInput); + new Regex("^([a-z]*)*$").Replace(userInput, ""); + // Known exponential blowup regex for e-mail address validation + // Problematic part is: ([a-zA-Z0-9]+))* + new Regex("^([a-zA-Z0-9])(([\\-.]|[_]+)?([a-zA-Z0-9]+))*(@){1}[a-z0-9]+[.]{1}(([a-z]{2,3})|([a-z]{2,3}[.]{1}[a-z]{2,3}))$").Match(userInput); + // Known exponential blowup regex for Java class name validation + // Problematic part is: (([a-z])+.)+ + new Regex(JAVA_CLASS_REGEX).Match(userInput); + // Static use + Regex.Match(userInput, JAVA_CLASS_REGEX); + // GOOD: + new Regex("^(([a-b]+[c-z]+)+$").Match(userInput); + new Regex("^([a-z]+)+$", RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1)).Match(userInput); + Regex.Match(userInput, JAVA_CLASS_REGEX, RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1)); + // Known possible FP. + new Regex("^[a-z0-9]+([_.-][a-z0-9]+)*$").Match(userInput); + } +} + +// The only purpose of this class is to make sure the extractor extracts the +// relevant library methods +public class LibraryTypeDataFlow +{ + void M() + { + int i; + int.Parse(""); + int.TryParse("", out i); + + bool b; + bool.Parse(""); + bool.TryParse("", out b); + + Uri uri = null; + uri.ToString(); + + StringReader sr = new StringReader(""); + + string s = new string(new[] { 'a' }); + string.Join("", "", "", ""); + + StringBuilder sb = new StringBuilder(""); + + Lazy l = new Lazy(() => 42); + + IEnumerable ie = null; + ie.GetEnumerator(); + ie.AsParallel(); + ie.AsQueryable(); + IEnumerable ieint = null; + ieint.Select(x => x); + List list = null; + list.Find(x => x > 0); + Stack stack = null; + stack.Peek(); + ArrayList al = null; + ArrayList.FixedSize(al); + SortedList sl = null; + sl.GetByIndex(0); + + Convert.ToInt32("0"); + + DataContract dc = null; + s = dc.AString; + + KeyValuePair kvp = new KeyValuePair(0, ""); + + IEnumerator ienum = null; + object o = ienum.Current; + + IEnumerator ienumint = null; + i = ienumint.Current; + + var task = new Task(() => { }); + Task.WhenAll(null, null); + Task.WhenAny(null, null); + Task.Factory.ContinueWhenAll((Task[])null, (Func)null); + + var task2 = new Task(() => 42); + Task.Factory.ContinueWhenAny(new Task[] { task2 }, t => t.Result.ToString()); + + Encoding.Unicode.GetString(Encoding.Unicode.GetBytes("")); + + Path.Combine("", ""); + Path.GetDirectoryName(""); + Path.GetExtension(""); + Path.GetFileName(""); + Path.GetFileNameWithoutExtension(""); + Path.GetPathRoot(""); + HttpContextBase context = null; + string name = context.Request.QueryString["name"]; + } + + [DataContract] + public class DataContract + { + [DataMember] + public string AString { get; set; } + } +} From abe5d0dd724280a34b84b108f4b0810078347f2a Mon Sep 17 00:00:00 2001 From: calum Date: Fri, 21 Sep 2018 13:06:33 +0100 Subject: [PATCH 06/58] C#: Fixes to stub generation. --- csharp/ql/src/Stubs/MinimalStubsFromSource.ql | 4 ++ csharp/ql/src/Stubs/Stubs.qll | 49 ++++++++++++++----- .../Stubs/MinimalStubsFromSource.expected | 2 +- 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/csharp/ql/src/Stubs/MinimalStubsFromSource.ql b/csharp/ql/src/Stubs/MinimalStubsFromSource.ql index e8751ded952..3a74abf8ae4 100644 --- a/csharp/ql/src/Stubs/MinimalStubsFromSource.ql +++ b/csharp/ql/src/Stubs/MinimalStubsFromSource.ql @@ -22,6 +22,10 @@ class UsedInSource extends GeneratedDeclaration { exists(Variable v | v.fromSource() | this = v.getType()) or exists(Virtualizable v | v.fromSource() | this = v.getImplementee() or this = v.getOverridee()) + or + this = any(Attribute a).getType() + or + this = any(Attribute a).getType().getAConstructor() ) and this.fromLibrary() diff --git a/csharp/ql/src/Stubs/Stubs.qll b/csharp/ql/src/Stubs/Stubs.qll index c7a971e166f..4c9d67a9d4f 100644 --- a/csharp/ql/src/Stubs/Stubs.qll +++ b/csharp/ql/src/Stubs/Stubs.qll @@ -117,9 +117,10 @@ private abstract class GeneratedType extends ValueOrRefType, GeneratedElement { private ValueOrRefType getAnInterestingBaseType() { result = this.getABaseType() and - not result instanceof ObjectType + not result instanceof ObjectType and + not result.getQualifiedName() = "System.ValueType" } - + private string stubBaseTypesString() { if this instanceof Enum then result = "" @@ -132,7 +133,7 @@ private abstract class GeneratedType extends ValueOrRefType, GeneratedElement { else result = "" } - + private string stubMembers() { result = concat(stubMember(this.getAGeneratedMember())) } @@ -171,15 +172,15 @@ private class IndirectType extends GeneratedType { or this.(UnboundGenericType).getAConstructedGeneric().getASubType() instanceof GeneratedType or - exists(GeneratedType t | this = getAContainedType(t.getAGeneratedType())) + exists(GeneratedType t | this = getAContainedType(t.getAGeneratedType()).getSourceDeclaration()) or - exists(GeneratedDeclaration decl | decl.(Member).getDeclaringType() = this) + exists(GeneratedDeclaration decl | decl.(Member).getDeclaringType().getSourceDeclaration() = this) } } private class RootGeneratedType extends GeneratedType { RootGeneratedType() { - this instanceof GeneratedDeclaration + this = any(GeneratedDeclaration decl).getSourceDeclaration() } } @@ -191,7 +192,7 @@ private Type getAContainedType(Type t) { private class RootGeneratedMember extends GeneratedMember { RootGeneratedMember() { - this instanceof GeneratedDeclaration + this = any(GeneratedDeclaration d).getSourceDeclaration() } } @@ -209,7 +210,7 @@ private class InheritedMember extends GeneratedMember, Virtualizable { or declarationExists(this.getOverridee+()) or - declarationExists(this.getAnOverrider*()) + declarationExists(this.getAnOverrider+()) } } @@ -328,6 +329,8 @@ private string stubClassName(Type t) { result = "float" else if t instanceof DoubleType then result = "double" + else if t instanceof NullableType then + result = stubClassName(t.(NullableType).getUnderlyingType()) + "?" else if t instanceof TypeParameter then result = t.getName() else if t instanceof ArrayType then @@ -345,9 +348,9 @@ private string stubClassName(Type t) { language[monotonicAggregates] private string stubGenericArguments(ValueOrRefType t) { if t instanceof UnboundGenericType then - result = "<" + concat(int n | exists(t.(UnboundGenericType).getTypeParameter(n)) | t.(UnboundGenericType).getTypeParameter(n).getName(),",") + ">" + result = "<" + concat(int n | exists(t.(UnboundGenericType).getTypeParameter(n)) | t.(UnboundGenericType).getTypeParameter(n).getName(),"," order by n) + ">" else if t instanceof ConstructedType then - result = "<" + concat(int n | exists(t.(ConstructedType).getTypeArgument(n)) | stubClassName(t.(ConstructedType).getTypeArgument(n)),",") + ">" + result = "<" + concat(int n | exists(t.(ConstructedType).getTypeArgument(n)) | stubClassName(t.(ConstructedType).getTypeArgument(n)),"," order by n) + ">" else result = "" } @@ -398,11 +401,17 @@ private string stubMember(Member m) { "(" + stubParameters(c) + ")" + stubImplementation(c) + ";\n" ) or - exists(Operator op | m = op and not m.getDeclaringType() instanceof Enum | + exists(Operator op | m = op and not m.getDeclaringType() instanceof Enum and not op instanceof ConversionOperator | result = " " + stubModifiers(op) + stubClassName(op.getReturnType()) + " operator " + op.getName() +"(" + stubParameters(op) + ") => throw null;\n" ) or + exists(ConversionOperator op | m = op | + result = " " + stubModifiers(op) + stubExplicit(op) + "operator " + + stubClassName(op.getReturnType()) + "(" + stubParameters(op) + + ") => throw null;\n" + ) + or result = " " + m.(EnumConstant).getName() + ",\n" or exists(Property p | m = p | @@ -422,6 +431,12 @@ private string stubMember(Member m) { ) } +private string stubExplicit(ConversionOperator op) { + op instanceof ImplicitConversionOperator and result = "implicit " + or + op instanceof ExplicitConversionOperator and result = "explicit " +} + private string stubGetter(DeclarationWithGetSetAccessors p) { if exists(p.getGetter()) then ( if p.isAbstract() or p.getDeclaringType() instanceof Interface then @@ -442,7 +457,17 @@ private string stubSetter(DeclarationWithGetSetAccessors p) { result = "" } +private string stubSemmleExtractorOptions() { + result = concat(string s | + exists(CommentLine comment | + s = "// original-extractor-options:" + comment.getText().regexpCapture("\\w*semmle-extractor-options:(.*)", 1) + "\n" + ) + ) +} + /** Gets the generated C# code. */ string generatedCode() { - result = any(GeneratedNamespace ns | ns.isGlobalNamespace()).getStubs() + result = "// This file contains auto-generated code.\n" + + stubSemmleExtractorOptions() + "\n" + + any(GeneratedNamespace ns | ns.isGlobalNamespace()).getStubs() } diff --git a/csharp/ql/test/query-tests/Stubs/MinimalStubsFromSource.expected b/csharp/ql/test/query-tests/Stubs/MinimalStubsFromSource.expected index f1b5fd84ab7..f049392cfcc 100644 --- a/csharp/ql/test/query-tests/Stubs/MinimalStubsFromSource.expected +++ b/csharp/ql/test/query-tests/Stubs/MinimalStubsFromSource.expected @@ -1 +1 @@ -| namespace System\n{\n// Generated from `System.Uri` in `System.Private.Uri, Version=4.0.4.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class Uri : System.Runtime.Serialization.ISerializable\n{\n public override bool Equals(object comparand) => throw null;\n public override int GetHashCode() => throw null;\n public override string ToString() => throw null;\n void System.Runtime.Serialization.ISerializable.GetObjectData(System.Runtime.Serialization.SerializationInfo serializationInfo, System.Runtime.Serialization.StreamingContext streamingContext) => throw null;\n}\n\nnamespace Collections\n{\n// Generated from `System.Collections.ArrayList` in `System.Runtime.Extensions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class ArrayList : System.ICloneable, System.Collections.IList, System.Collections.IEnumerable, System.Collections.ICollection\n{\n public static System.Collections.ArrayList FixedSize(System.Collections.ArrayList list) => throw null;\n public virtual System.Collections.IEnumerator GetEnumerator() => throw null;\n public virtual bool Contains(object item) => throw null;\n public virtual bool IsFixedSize { get => throw null; }\n public virtual bool IsReadOnly { get => throw null; }\n public virtual bool IsSynchronized { get => throw null; }\n public virtual int Add(object value) => throw null;\n public virtual int Count { get => throw null; }\n public virtual int IndexOf(object value) => throw null;\n public virtual object Clone() => throw null;\n public virtual object SyncRoot { get => throw null; }\n public virtual object this[int index] { get => throw null; set => throw null; }\n public virtual void Clear() => throw null;\n public virtual void CopyTo(System.Array array, int arrayIndex) => throw null;\n public virtual void Insert(int index, object value) => throw null;\n public virtual void Remove(object obj) => throw null;\n public virtual void RemoveAt(int index) => throw null;\n}\n\n// Generated from `System.Collections.SortedList` in `System.Collections.NonGeneric, Version=4.1.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class SortedList : System.ICloneable, System.Collections.IEnumerable, System.Collections.IDictionary, System.Collections.ICollection\n{\n System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => throw null;\n public virtual System.Collections.ICollection Keys { get => throw null; }\n public virtual System.Collections.ICollection Values { get => throw null; }\n public virtual System.Collections.IDictionaryEnumerator GetEnumerator() => throw null;\n public virtual bool Contains(object key) => throw null;\n public virtual bool IsFixedSize { get => throw null; }\n public virtual bool IsReadOnly { get => throw null; }\n public virtual bool IsSynchronized { get => throw null; }\n public virtual int Count { get => throw null; }\n public virtual object Clone() => throw null;\n public virtual object GetByIndex(int index) => throw null;\n public virtual object SyncRoot { get => throw null; }\n public virtual object this[object key] { get => throw null; set => throw null; }\n public virtual void Add(object key, object value) => throw null;\n public virtual void Clear() => throw null;\n public virtual void CopyTo(System.Array array, int arrayIndex) => throw null;\n public virtual void Remove(object key) => throw null;\n}\n\nnamespace Specialized\n{\n// Generated from `System.Collections.Specialized.NameObjectCollectionBase` in `System.Collections.Specialized, Version=4.1.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\nabstract public class NameObjectCollectionBase : System.Runtime.Serialization.ISerializable, System.Runtime.Serialization.IDeserializationCallback, System.Collections.IEnumerable, System.Collections.ICollection\n{\n bool System.Collections.ICollection.IsSynchronized { get => throw null; }\n object System.Collections.ICollection.SyncRoot { get => throw null; }\n public virtual System.Collections.IEnumerator GetEnumerator() => throw null;\n public virtual int Count { get => throw null; }\n public virtual void GetObjectData(System.Runtime.Serialization.SerializationInfo info, System.Runtime.Serialization.StreamingContext context) => throw null;\n public virtual void OnDeserialization(object sender) => throw null;\n void System.Collections.ICollection.CopyTo(System.Array array, int index) => throw null;\n}\n\n// Generated from `System.Collections.Specialized.NameValueCollection` in `System.Collections.Specialized, Version=4.1.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class NameValueCollection : System.Collections.Specialized.NameObjectCollectionBase\n{\n public string this[string name] { get => throw null; set => throw null; }\n}\n\n}\n}\nnamespace IO\n{\n// Generated from `System.IO.StringReader` in `System.Runtime.Extensions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class StringReader : System.IO.TextReader\n{\n public StringReader(string s) => throw null;\n}\n\n// Generated from `System.IO.TextReader` in `System.Runtime.Extensions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\nabstract public class TextReader : System.MarshalByRefObject, System.IDisposable\n{\n public void Dispose() => throw null;\n}\n\n}\nnamespace Linq\n{\n// Generated from `System.Linq.Enumerable` in `System.Linq, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\nstatic public class Enumerable\n{\n public static System.Collections.Generic.IEnumerable Select(this System.Collections.Generic.IEnumerable source, System.Func selector) => throw null;\n}\n\n// Generated from `System.Linq.IQueryable` in `System.Linq.Expressions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic interface IQueryable : System.Collections.IEnumerable\n{\n}\n\n// Generated from `System.Linq.ParallelEnumerable` in `System.Linq.Parallel, Version=4.0.3.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\nstatic public class ParallelEnumerable\n{\n public static System.Linq.ParallelQuery AsParallel(this System.Collections.IEnumerable source) => throw null;\n}\n\n// Generated from `System.Linq.ParallelQuery` in `System.Linq.Parallel, Version=4.0.3.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class ParallelQuery : System.Collections.IEnumerable\n{\n System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => throw null;\n}\n\n// Generated from `System.Linq.Queryable` in `System.Linq.Queryable, Version=4.0.3.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\nstatic public class Queryable\n{\n public static System.Linq.IQueryable AsQueryable(this System.Collections.IEnumerable source) => throw null;\n}\n\n}\nnamespace Text\n{\nnamespace RegularExpressions\n{\n// Generated from `System.Text.RegularExpressions.Capture` in `System.Text.RegularExpressions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class Capture\n{\n public override string ToString() => throw null;\n}\n\n// Generated from `System.Text.RegularExpressions.Group` in `System.Text.RegularExpressions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class Group : System.Text.RegularExpressions.Capture\n{\n}\n\n// Generated from `System.Text.RegularExpressions.Match` in `System.Text.RegularExpressions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class Match : System.Text.RegularExpressions.Group\n{\n}\n\n// Generated from `System.Text.RegularExpressions.RegexOptions` in `System.Text.RegularExpressions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic enum RegexOptions\n{\n IgnoreCase,\n}\n\n// Generated from `System.Text.RegularExpressions.Regex` in `System.Text.RegularExpressions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class Regex : System.Runtime.Serialization.ISerializable\n{\n public Regex(string pattern) => throw null;\n public Regex(string pattern, System.Text.RegularExpressions.RegexOptions options, System.TimeSpan matchTimeout) => throw null;\n public System.Text.RegularExpressions.Match Match(string input) => throw null;\n public override string ToString() => throw null;\n public static System.Text.RegularExpressions.Match Match(string input, string pattern) => throw null;\n public static System.Text.RegularExpressions.Match Match(string input, string pattern, System.Text.RegularExpressions.RegexOptions options, System.TimeSpan matchTimeout) => throw null;\n public string Replace(string input, string replacement) => throw null;\n void System.Runtime.Serialization.ISerializable.GetObjectData(System.Runtime.Serialization.SerializationInfo si, System.Runtime.Serialization.StreamingContext context) => throw null;\n}\n\n}\n}\n}\n | +| // This file contains auto-generated code.\n// original-extractor-options: /r:System.Text.RegularExpressions.dll /r:System.Collections.Specialized.dll /r:System.Net.dll /r:System.Web.dll /r:System.Net.HttpListener.dll /r:System.Collections.Specialized.dll /r:System.Private.Uri.dll /r:System.Runtime.Extensions.dll /r:System.Linq.Parallel.dll /r:System.Collections.Concurrent.dll /r:System.Linq.Expressions.dll /r:System.Collections.dll /r:System.Linq.Queryable.dll /r:System.Linq.dll /r:System.Collections.NonGeneric.dll /r:System.ObjectModel.dll /r:System.ComponentModel.TypeConverter.dll /r:System.IO.Compression.dll /r:System.IO.Pipes.dll /r:System.Net.Primitives.dll /r:System.Net.Security.dll /r:System.Security.Cryptography.Primitives.dll /r:System.Text.RegularExpressions.dll ${testdir}/../../resources/stubs/System.Web.cs /r:System.Runtime.Serialization.Primitives.dll\n\nnamespace System\n{\n// Generated from `System.Uri` in `System.Private.Uri, Version=4.0.4.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class Uri : System.Runtime.Serialization.ISerializable\n{\n public override bool Equals(object comparand) => throw null;\n public override int GetHashCode() => throw null;\n public override string ToString() => throw null;\n void System.Runtime.Serialization.ISerializable.GetObjectData(System.Runtime.Serialization.SerializationInfo serializationInfo, System.Runtime.Serialization.StreamingContext streamingContext) => throw null;\n}\n\nnamespace Collections\n{\n// Generated from `System.Collections.ArrayList` in `System.Runtime.Extensions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class ArrayList : System.ICloneable, System.Collections.IList, System.Collections.IEnumerable, System.Collections.ICollection\n{\n public static System.Collections.ArrayList FixedSize(System.Collections.ArrayList list) => throw null;\n public virtual System.Collections.IEnumerator GetEnumerator() => throw null;\n public virtual bool Contains(object item) => throw null;\n public virtual bool IsFixedSize { get => throw null; }\n public virtual bool IsReadOnly { get => throw null; }\n public virtual bool IsSynchronized { get => throw null; }\n public virtual int Add(object value) => throw null;\n public virtual int Count { get => throw null; }\n public virtual int IndexOf(object value) => throw null;\n public virtual object Clone() => throw null;\n public virtual object SyncRoot { get => throw null; }\n public virtual object this[int index] { get => throw null; set => throw null; }\n public virtual void Clear() => throw null;\n public virtual void CopyTo(System.Array array, int arrayIndex) => throw null;\n public virtual void Insert(int index, object value) => throw null;\n public virtual void Remove(object obj) => throw null;\n public virtual void RemoveAt(int index) => throw null;\n}\n\n// Generated from `System.Collections.SortedList` in `System.Collections.NonGeneric, Version=4.1.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class SortedList : System.ICloneable, System.Collections.IEnumerable, System.Collections.IDictionary, System.Collections.ICollection\n{\n System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => throw null;\n public virtual System.Collections.ICollection Keys { get => throw null; }\n public virtual System.Collections.ICollection Values { get => throw null; }\n public virtual System.Collections.IDictionaryEnumerator GetEnumerator() => throw null;\n public virtual bool Contains(object key) => throw null;\n public virtual bool IsFixedSize { get => throw null; }\n public virtual bool IsReadOnly { get => throw null; }\n public virtual bool IsSynchronized { get => throw null; }\n public virtual int Count { get => throw null; }\n public virtual object Clone() => throw null;\n public virtual object GetByIndex(int index) => throw null;\n public virtual object SyncRoot { get => throw null; }\n public virtual object this[object key] { get => throw null; set => throw null; }\n public virtual void Add(object key, object value) => throw null;\n public virtual void Clear() => throw null;\n public virtual void CopyTo(System.Array array, int arrayIndex) => throw null;\n public virtual void Remove(object key) => throw null;\n}\n\nnamespace Generic\n{\n// Generated from `System.Collections.Generic.Stack<>` in `System.Collections, Version=4.1.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class Stack : System.Collections.IEnumerable, System.Collections.ICollection, System.Collections.Generic.IReadOnlyCollection, System.Collections.Generic.IEnumerable\n{\n System.Collections.Generic.IEnumerator System.Collections.Generic.IEnumerable.GetEnumerator() => throw null;\n System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => throw null;\n bool System.Collections.ICollection.IsSynchronized { get => throw null; }\n object System.Collections.ICollection.SyncRoot { get => throw null; }\n public T Peek() => throw null;\n public int Count { get => throw null; }\n void System.Collections.ICollection.CopyTo(System.Array array, int arrayIndex) => throw null;\n}\n\n}\nnamespace Specialized\n{\n// Generated from `System.Collections.Specialized.NameObjectCollectionBase` in `System.Collections.Specialized, Version=4.1.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\nabstract public class NameObjectCollectionBase : System.Runtime.Serialization.ISerializable, System.Runtime.Serialization.IDeserializationCallback, System.Collections.IEnumerable, System.Collections.ICollection\n{\n bool System.Collections.ICollection.IsSynchronized { get => throw null; }\n object System.Collections.ICollection.SyncRoot { get => throw null; }\n public virtual System.Collections.IEnumerator GetEnumerator() => throw null;\n public virtual int Count { get => throw null; }\n public virtual void GetObjectData(System.Runtime.Serialization.SerializationInfo info, System.Runtime.Serialization.StreamingContext context) => throw null;\n public virtual void OnDeserialization(object sender) => throw null;\n void System.Collections.ICollection.CopyTo(System.Array array, int index) => throw null;\n}\n\n// Generated from `System.Collections.Specialized.NameValueCollection` in `System.Collections.Specialized, Version=4.1.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class NameValueCollection : System.Collections.Specialized.NameObjectCollectionBase\n{\n public string this[string name] { get => throw null; set => throw null; }\n}\n\n}\n}\nnamespace IO\n{\n// Generated from `System.IO.StringReader` in `System.Runtime.Extensions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class StringReader : System.IO.TextReader\n{\n public StringReader(string s) => throw null;\n}\n\n// Generated from `System.IO.TextReader` in `System.Runtime.Extensions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\nabstract public class TextReader : System.MarshalByRefObject, System.IDisposable\n{\n public void Dispose() => throw null;\n}\n\n}\nnamespace Linq\n{\n// Generated from `System.Linq.Enumerable` in `System.Linq, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\nstatic public class Enumerable\n{\n public static System.Collections.Generic.IEnumerable Select(this System.Collections.Generic.IEnumerable source, System.Func selector) => throw null;\n}\n\n// Generated from `System.Linq.IQueryable` in `System.Linq.Expressions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic interface IQueryable : System.Collections.IEnumerable\n{\n}\n\n// Generated from `System.Linq.ParallelEnumerable` in `System.Linq.Parallel, Version=4.0.3.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\nstatic public class ParallelEnumerable\n{\n public static System.Linq.ParallelQuery AsParallel(this System.Collections.IEnumerable source) => throw null;\n}\n\n// Generated from `System.Linq.ParallelQuery` in `System.Linq.Parallel, Version=4.0.3.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class ParallelQuery : System.Collections.IEnumerable\n{\n System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => throw null;\n}\n\n// Generated from `System.Linq.Queryable` in `System.Linq.Queryable, Version=4.0.3.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\nstatic public class Queryable\n{\n public static System.Linq.IQueryable AsQueryable(this System.Collections.IEnumerable source) => throw null;\n}\n\n}\nnamespace Runtime\n{\nnamespace Serialization\n{\n// Generated from `System.Runtime.Serialization.DataContractAttribute` in `System.Runtime.Serialization.Primitives, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class DataContractAttribute : System.Attribute\n{\n public DataContractAttribute() => throw null;\n}\n\n// Generated from `System.Runtime.Serialization.DataMemberAttribute` in `System.Runtime.Serialization.Primitives, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class DataMemberAttribute : System.Attribute\n{\n public DataMemberAttribute() => throw null;\n}\n\n}\n}\nnamespace Text\n{\nnamespace RegularExpressions\n{\n// Generated from `System.Text.RegularExpressions.Capture` in `System.Text.RegularExpressions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class Capture\n{\n public override string ToString() => throw null;\n}\n\n// Generated from `System.Text.RegularExpressions.Group` in `System.Text.RegularExpressions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class Group : System.Text.RegularExpressions.Capture\n{\n}\n\n// Generated from `System.Text.RegularExpressions.Match` in `System.Text.RegularExpressions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class Match : System.Text.RegularExpressions.Group\n{\n}\n\n// Generated from `System.Text.RegularExpressions.RegexOptions` in `System.Text.RegularExpressions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic enum RegexOptions\n{\n IgnoreCase,\n}\n\n// Generated from `System.Text.RegularExpressions.Regex` in `System.Text.RegularExpressions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a`\npublic class Regex : System.Runtime.Serialization.ISerializable\n{\n public Regex(string pattern) => throw null;\n public Regex(string pattern, System.Text.RegularExpressions.RegexOptions options, System.TimeSpan matchTimeout) => throw null;\n public System.Text.RegularExpressions.Match Match(string input) => throw null;\n public override string ToString() => throw null;\n public static System.Text.RegularExpressions.Match Match(string input, string pattern) => throw null;\n public static System.Text.RegularExpressions.Match Match(string input, string pattern, System.Text.RegularExpressions.RegexOptions options, System.TimeSpan matchTimeout) => throw null;\n public string Replace(string input, string replacement) => throw null;\n void System.Runtime.Serialization.ISerializable.GetObjectData(System.Runtime.Serialization.SerializationInfo si, System.Runtime.Serialization.StreamingContext context) => throw null;\n}\n\n}\n}\n}\n | From d813cb63e7684a31b64f9505574606d337614a21 Mon Sep 17 00:00:00 2001 From: Jonas Jensen Date: Wed, 26 Sep 2018 15:29:07 +0200 Subject: [PATCH 07/58] C++: Upper-case Boolean and around HRESULT --- .../Security/CWE/CWE-253/HResultBooleanConversion.qhelp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/ql/src/Security/CWE/CWE-253/HResultBooleanConversion.qhelp b/cpp/ql/src/Security/CWE/CWE-253/HResultBooleanConversion.qhelp index d5b949691f0..fdc31608922 100644 --- a/cpp/ql/src/Security/CWE/CWE-253/HResultBooleanConversion.qhelp +++ b/cpp/ql/src/Security/CWE/CWE-253/HResultBooleanConversion.qhelp @@ -4,13 +4,13 @@ -

This query indicates that an HRESULT is being cast to a boolean type or vice versa.

-

The typical success value (S_OK) of an HRESULT equals 0. However, 0 indicates failure for a boolean type.

-

Casting an HRESULT to a boolean type and then using it in a test expression will yield an incorrect result.

+

This query indicates that an HRESULT is being cast to a Boolean type or vice versa.

+

The typical success value (S_OK) of an HRESULT equals 0. However, 0 indicates failure for a Boolean type.

+

Casting an HRESULT to a Boolean type and then using it in a test expression will yield an incorrect result.

 -

To check if a call that returns an HRESULT succeeded use the FAILED macro.

+

To check if a call that returns an HRESULT succeeded use the FAILED macro.

 From 532a64f211b6b328966ee6088f3d573c1658eae5 Mon Sep 17 00:00:00 2001 From: Jonas Jensen Date: Mon, 1 Oct 2018 12:12:00 +0200 Subject: [PATCH 08/58] C++: Name/description of HResultBooleanConversion This commit changes the name and description of the new `HResultBooleanConversion` query to follow our internal guidelines. --- .../src/Security/CWE/CWE-253/HResultBooleanConversion.ql | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cpp/ql/src/Security/CWE/CWE-253/HResultBooleanConversion.ql b/cpp/ql/src/Security/CWE/CWE-253/HResultBooleanConversion.ql index 7678cff23f0..165fe0eed96 100644 --- a/cpp/ql/src/Security/CWE/CWE-253/HResultBooleanConversion.ql +++ b/cpp/ql/src/Security/CWE/CWE-253/HResultBooleanConversion.ql @@ -1,8 +1,6 @@ /** - * @name Cast between semantically different integer types: HRESULT to/from a Boolean type - * @description Cast between semantically different integer types: HRESULT to/from a Boolean type. - * Boolean types indicate success by a non-zero value, whereas success (S_OK) in HRESULT is indicated by a value of 0. - * Casting an HRESULT to/from a Boolean type and then using it in a test expression will yield an incorrect result. + * @name Cast between HRESULT and a Boolean type + * @description Casting an HRESULT to/from a Boolean type and then using it in a test expression will yield an incorrect result because success (S_OK) in HRESULT is indicated by a value of 0. * @kind problem * @id cpp/hresult-boolean-conversion * @problem.severity error @@ -68,4 +66,4 @@ where exists ) and not isHresultBooleanConverted(e1) ) -select e1, msg \ No newline at end of file +select e1, msg From 54cd173da8ff558f6b70b7896544cf1ae5f674e5 Mon Sep 17 00:00:00 2001 From: Jonas Jensen Date: Mon, 1 Oct 2018 12:13:14 +0200 Subject: [PATCH 09/58] C++: Changelog entries for two new queries --- change-notes/1.19/analysis-cpp.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/change-notes/1.19/analysis-cpp.md b/change-notes/1.19/analysis-cpp.md index 5a3340603c8..f45a4ee92ea 100644 --- a/change-notes/1.19/analysis-cpp.md +++ b/change-notes/1.19/analysis-cpp.md @@ -6,7 +6,8 @@ | **Query** | **Tags** | **Purpose** | |-----------------------------|-----------|--------------------------------------------------------------------| -| *@name of query (Query ID)* | *Tags* |*Aim of the new query and whether it is enabled by default or not* | +| Cast between HRESULT and a Boolean type (`cpp/hresult-boolean-conversion`) | external/cwe/cwe-253 | Finds logic errors caused by mistakenly treating the Windows `HRESULT` type as a Boolean instead of testing it with the appropriate macros. Enabled by default. | +| Setting a DACL to `NULL` in a `SECURITY_DESCRIPTOR` (`cpp/unsafe-dacl-security-descriptor`) | external/cwe/cwe-732 | This query finds code that creates world-writable objects on Windows by setting their DACL to `NULL`. Enabled by default. | ## Changes to existing queries From 308631e8ff45e8f6b2a771a50f9087985d41bf4f Mon Sep 17 00:00:00 2001 From: Jonas Jensen Date: Mon, 1 Oct 2018 13:38:13 +0200 Subject: [PATCH 10/58] C++: Add two recent queries to query suites --- cpp/config/suites/c/correctness | 1 + cpp/config/suites/cpp/correctness | 1 + cpp/config/suites/security/cwe-253 | 3 +++ cpp/config/suites/security/cwe-732 | 2 ++ cpp/config/suites/security/default | 1 + 5 files changed, 8 insertions(+) create mode 100644 cpp/config/suites/security/cwe-253 diff --git a/cpp/config/suites/c/correctness b/cpp/config/suites/c/correctness index f3f9bebf4a5..e53ccda1187 100644 --- a/cpp/config/suites/c/correctness +++ b/cpp/config/suites/c/correctness @@ -6,6 +6,7 @@ + semmlecode-cpp-queries/Likely Bugs/Arithmetic/IntMultToLong.ql: /Correctness/Dangerous Conversions + semmlecode-cpp-queries/Likely Bugs/Conversion/NonzeroValueCastToPointer.ql: /Correctness/Dangerous Conversions + semmlecode-cpp-queries/Likely Bugs/Conversion/ImplicitDowncastFromBitfield.ql: /Correctness/Dangerous Conversions ++ semmlecode-cpp-queries/Security/CWE/CWE-253/HResultBooleanConversion.ql: /Correctness/Dangerous Conversions # Consistent Use + semmlecode-cpp-queries/Critical/ReturnValueIgnored.ql: /Correctness/Consistent Use + semmlecode-cpp-queries/Likely Bugs/InconsistentCheckReturnNull.ql: /Correctness/Consistent Use diff --git a/cpp/config/suites/cpp/correctness b/cpp/config/suites/cpp/correctness index e1195442623..ceb15f55260 100644 --- a/cpp/config/suites/cpp/correctness +++ b/cpp/config/suites/cpp/correctness @@ -7,6 +7,7 @@ + semmlecode-cpp-queries/Likely Bugs/Conversion/NonzeroValueCastToPointer.ql: /Correctness/Dangerous Conversions + semmlecode-cpp-queries/Likely Bugs/Conversion/ImplicitDowncastFromBitfield.ql: /Correctness/Dangerous Conversions + semmlecode-cpp-queries/Likely Bugs/Conversion/CastArrayPointerArithmetic.ql: /Correctness/Dangerous Conversions ++ semmlecode-cpp-queries/Security/CWE/CWE-253/HResultBooleanConversion.ql: /Correctness/Dangerous Conversions # Consistent Use + semmlecode-cpp-queries/Critical/ReturnValueIgnored.ql: /Correctness/Consistent Use + semmlecode-cpp-queries/Likely Bugs/InconsistentCheckReturnNull.ql: /Correctness/Consistent Use diff --git a/cpp/config/suites/security/cwe-253 b/cpp/config/suites/security/cwe-253 new file mode 100644 index 00000000000..1ce40c4e71a --- /dev/null +++ b/cpp/config/suites/security/cwe-253 @@ -0,0 +1,3 @@ +# CWE-253: Incorrect Check of Function Return Value ++ semmlecode-cpp-queries/Security/CWE/CWE-253/HResultBooleanConversion.ql: /CWE/CWE-253 + @name Cast between HRESULT and a Boolean type (CWE-253) diff --git a/cpp/config/suites/security/cwe-732 b/cpp/config/suites/security/cwe-732 index 90af4998438..77de440b90e 100644 --- a/cpp/config/suites/security/cwe-732 +++ b/cpp/config/suites/security/cwe-732 @@ -1,3 +1,5 @@ # CWE-732: Incorrect Permission Assignment for Critical Resource + semmlecode-cpp-queries/Security/CWE/CWE-732/DoNotCreateWorldWritable.ql: /CWE/CWE-732 @name File created without restricting permissions (CWE-732) ++ semmlecode-cpp-queries/Security/CWE/CWE-732/UnsafeDaclSecurityDescriptor.ql: /CWE/CWE-732 + @name Setting a DACL to NULL in a SECURITY_DESCRIPTOR (CWE-732) diff --git a/cpp/config/suites/security/default b/cpp/config/suites/security/default index 4225ae9808b..baa8a3f5bc8 100644 --- a/cpp/config/suites/security/default +++ b/cpp/config/suites/security/default @@ -13,6 +13,7 @@ @import "cwe-170" @import "cwe-190" @import "cwe-242" +@import "cwe-253" @import "cwe-290" @import "cwe-311" @import "cwe-327" From 1a90f7df2c18cb41c949b3dba60b9499246939cc Mon Sep 17 00:00:00 2001 From: Luke Cartey Date: Wed, 3 Oct 2018 11:38:48 +0100 Subject: [PATCH 11/58] C#: ZipSlip - Address review comments. - Add backticks - Add extra test. --- .../semmle/code/csharp/security/dataflow/ZipSlip.qll | 2 +- .../Security Features/CWE-022/ZipSlip/ZipSlip.cs | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/csharp/ql/src/semmle/code/csharp/security/dataflow/ZipSlip.qll b/csharp/ql/src/semmle/code/csharp/security/dataflow/ZipSlip.qll index a58d131e0f7..403648b9891 100644 --- a/csharp/ql/src/semmle/code/csharp/security/dataflow/ZipSlip.qll +++ b/csharp/ql/src/semmle/code/csharp/security/dataflow/ZipSlip.qll @@ -112,7 +112,7 @@ module ZipSlip { } /** - * A call to Substring. + * A call to `Substring`. * * This is considered a sanitizer because `Substring` may be used to extract a single component * of a path to avoid ZipSlip. diff --git a/csharp/ql/test/query-tests/Security Features/CWE-022/ZipSlip/ZipSlip.cs b/csharp/ql/test/query-tests/Security Features/CWE-022/ZipSlip/ZipSlip.cs index d82f76ced0a..22eb61f8c83 100644 --- a/csharp/ql/test/query-tests/Security Features/CWE-022/ZipSlip/ZipSlip.cs +++ b/csharp/ql/test/query-tests/Security Features/CWE-022/ZipSlip/ZipSlip.cs @@ -59,7 +59,7 @@ namespace ZipSlip foreach (ZipArchiveEntry entry in archive.Entries) { // figure out where we are putting the file - string destFilePath = Path.Combine(InstallDir, entry.FullName); + String destFilePath = Path.Combine(InstallDir, entry.FullName); Directory.CreateDirectory(Path.GetDirectoryName(destFilePath)); @@ -94,6 +94,15 @@ namespace ZipSlip Console.WriteLine(@"Writing ""{0}""", destFilePath); archiveFileStream.CopyTo(fs); } + + // GOOD: Use substring to pick out single component + string fileName = destFilePath.Substring(destFilePath.LastIndexOf("\\")); + var fileInfo2 = new FileInfo(fileName); + using (FileStream fs = fileInfo2.Open(FileMode.Create)) + { + Console.WriteLine(@"Writing ""{0}""", destFilePath); + archiveFileStream.CopyTo(fs); + } } } } From 19215d086802aa6643a0238a2bc83e4b52d98d2d Mon Sep 17 00:00:00 2001 From: Tom Hvitved Date: Tue, 2 Oct 2018 12:48:57 +0200 Subject: [PATCH 12/58] C#: Improve performance of type conversion library --- .../ql/src/semmle/code/csharp/Conversion.qll | 616 ++++++++++++------ 1 file changed, 412 insertions(+), 204 deletions(-) diff --git a/csharp/ql/src/semmle/code/csharp/Conversion.qll b/csharp/ql/src/semmle/code/csharp/Conversion.qll index 332d290f628..bb4c697d7dc 100644 --- a/csharp/ql/src/semmle/code/csharp/Conversion.qll +++ b/csharp/ql/src/semmle/code/csharp/Conversion.qll @@ -64,6 +64,20 @@ private Type getTypeArgument(UnboundGenericType ugt, ConstructedType ct, int i, result = ct.getTypeArgument(i) } +/** A type that is an element type of an array type. */ +private class ArrayElementType extends Type { + ArrayElementType() { + this = any(ArrayType at).getElementType() + } +} + +/** A type that is an argument in a constructed type. */ +private class TypeArgument extends Type { + TypeArgument() { + this = any(ConstructedType ct).getATypeArgument() + } +} + /** * INTERNAL: Do not use. * @@ -78,125 +92,213 @@ private Type getTypeArgument(UnboundGenericType ugt, ConstructedType ct, int i, predicate convIdentity(Type fromType, Type toType) { fromType = toType or - convIdentityStrict(fromType, toType) + Identity::convIdentityStrict(fromType, toType) } -private class IdentityConvertibleType extends Type { - IdentityConvertibleType() { - isIdentityConvertible(this) +private module Identity { + private class IdentityConvertibleType extends Type { + IdentityConvertibleType() { + isIdentityConvertible(this) + } } -} -private class IdentityConvertibleArrayType extends IdentityConvertibleType, ArrayType { } + private class IdentityConvertibleArrayType extends IdentityConvertibleType, ArrayType { } -private class IdentityConvertibleConstructedType extends IdentityConvertibleType { - IdentityConvertibleConstructedType() { - this instanceof ConstructedType - } -} + private class IdentityConvertibleConstructedType extends IdentityConvertibleType, ConstructedType { } -/** - * A type is (strictly) identity convertible if it contains at least one `object` - * or one `dynamic` sub term. - */ -private predicate isIdentityConvertible(Type t) { - t instanceof ObjectType - or - t instanceof DynamicType - or - isIdentityConvertible(t.(ArrayType).getElementType()) - or - isIdentityConvertible(t.(ConstructedType).getATypeArgument()) -} - -private predicate convIdentityStrict(IdentityConvertibleType fromType, IdentityConvertibleType toType) { - convIdentityObjectDynamic(fromType, toType) - or - convIdentityObjectDynamic(toType, fromType) - or - convIdentityStrictArrayType(fromType, toType) - or - convIdentityStrictConstructedType(fromType, toType) -} - -private predicate convIdentityObjectDynamic(ObjectType fromType, DynamicType toType) { - any() -} - -private predicate convIdentityStrictArrayType(IdentityConvertibleArrayType fromType, IdentityConvertibleArrayType toType) { - convIdentityStrictArrayTypeJoin(fromType, toType, toType.getDimension(), toType.getRank()) -} - -pragma [noinline] -private predicate convIdentityStrictArrayTypeJoin(IdentityConvertibleArrayType fromType, IdentityConvertibleArrayType toType, int dim, int rnk) { - convIdentityStrictArrayElementType(fromType, toType.getElementType(), dim, rnk) -} - -private predicate convIdentityStrictArrayElementType(IdentityConvertibleArrayType fromType, ArrayElementType aet, int dim, int rnk) { - convIdentityStrict(fromType.getElementType(), aet) and - dim = fromType.getDimension() and - rnk = fromType.getRank() -} - -/** A type that is an element type of an array type. */ -private class ArrayElementType extends Type { - ArrayElementType() { - this = any(ArrayType at).getElementType() - } -} - -private predicate convIdentityStrictConstructedType(IdentityConvertibleConstructedType fromType, IdentityConvertibleConstructedType toType) { - /* Semantically equivalent with - * ``` - * ugt = fromType.getUnboundGeneric() - * and - * forex(int i | - * i in [0 .. ugt.getNumberOfTypeParameters() - 1] | - * exists(Type t1, Type t2 | - * t1 = getTypeArgument(ugt, fromType, i, _) and - * t2 = getTypeArgument(ugt, toType, i, _) | - * convIdentity(t1, t2) - * ) - * ) - * ``` - * but performance is improved by explicitly evaluating the `i`th argument - * only when all preceding arguments are convertible. + /** + * A type is (strictly) identity convertible if it contains at least one `object` + * or one `dynamic` sub term. */ - exists(UnboundGenericType ugt | - convIdentityStrictConstructedTypeFromZero(ugt, fromType, toType, ugt.getNumberOfTypeParameters() - 1) - ) -} - -/** - * Holds if the type arguments 0 through `i` of `fromType` and `toType` are identity convertible. - */ -private predicate convIdentityStrictConstructedTypeFromZero(UnboundGenericType ugt, IdentityConvertibleConstructedType fromType, IdentityConvertibleConstructedType toType, int i) { - exists(Type toTypeArgument | - convIdentityStrictConstructedTypeFromZeroAux(ugt, fromType, i, toTypeArgument) and - toTypeArgument = getTypeArgument(ugt, toType, i, _) and - fromType != toType | - i = 0 + private predicate isIdentityConvertible(Type t) { + t instanceof ObjectType or - convIdentityStrictConstructedTypeFromZero(ugt, fromType, toType, i - 1) - ) -} + t instanceof DynamicType + or + isIdentityConvertible(t.(ArrayType).getElementType()) + or + isIdentityConvertible(t.(ConstructedType).getATypeArgument()) + } -pragma [nomagic] -private predicate convIdentityStrictConstructedTypeFromZeroAux(UnboundGenericType ugt, IdentityConvertibleConstructedType fromType, int i, Type toTypeArgument) { - exists(Type fromTypeArgument | - fromTypeArgument = getTypeArgument(ugt, fromType, i, _) and - convIdentityTypeArgument(fromTypeArgument, toTypeArgument) - ) -} + predicate convIdentityStrict(IdentityConvertibleType fromType, IdentityConvertibleType toType) { + convIdentityObjectDynamic(fromType, toType) + or + convIdentityObjectDynamic(toType, fromType) + or + convIdentityStrictArrayType(fromType, toType) + or + convIdentityStrictConstructedType(fromType, toType) + } -private predicate convIdentityTypeArgument(TypeArgument fromType, TypeArgument toType) { - convIdentity(fromType, toType) -} + private predicate convIdentityObjectDynamic(ObjectType fromType, DynamicType toType) { + any() + } -/** A type that is an argument in a constructed type. */ -private class TypeArgument extends Type { - TypeArgument() { - this = any(ConstructedType ct).getATypeArgument() + private predicate convIdentityStrictArrayType(IdentityConvertibleArrayType fromType, IdentityConvertibleArrayType toType) { + convIdentityStrictArrayTypeJoin(fromType, toType, toType.getDimension(), toType.getRank()) + } + + pragma[noinline] + private predicate convIdentityStrictArrayTypeJoin(IdentityConvertibleArrayType fromType, IdentityConvertibleArrayType toType, int dim, int rnk) { + convIdentityStrictArrayElementType(fromType, toType.getElementType(), dim, rnk) + } + + private predicate convIdentityStrictArrayElementType(IdentityConvertibleArrayType fromType, ArrayElementType aet, int dim, int rnk) { + convIdentityStrict(fromType.getElementType(), aet) and + dim = fromType.getDimension() and + rnk = fromType.getRank() + } + + /** + * Gets the number of different type arguments supplied for the type + * parameter at index `i` in unbound generic type `ugt`. + */ + private int getTypeArgumentCount(UnboundGenericType ugt, int i) { + result = strictcount(Type arg | + exists(IdentityConvertibleConstructedType ct | + ct.getUnboundGeneric() = ugt | + arg = ct.getTypeArgument(i) + ) + ) + } + + private int rnk(UnboundGenericType ugt, int i) { + result = rank[i + 1](int j, int k | + j = getTypeArgumentCount(ugt, k) | + k order by j, k + ) + } + + private Type getTypeArgumentRanked(UnboundGenericType ugt, IdentityConvertibleConstructedType t, int i) { + result = getTypeArgument(ugt, t, rnk(ugt, i), _) + } + + /** + * Holds if `fromTypeArgument` is identity convertible to `toTypeArgument`, and + * both types are the `i`th type argument in _some_ constructed type. + */ + pragma[nomagic] + private predicate convTypeArguments(Type fromTypeArgument, Type toTypeArgument, int i) { + exists(int j | + fromTypeArgument = getTypeArgumentRanked(_, _, i) and + toTypeArgument = getTypeArgumentRanked(_, _, j) and + i <= j and j <= i + | + convIdentity(fromTypeArgument, toTypeArgument) + ) + } + + pragma[nomagic] + private predicate convTypeArgumentsSomeUnbound(UnboundGenericType ugt, TypeArgument fromTypeArgument, TypeArgument toTypeArgument, int i) { + convTypeArguments(fromTypeArgument, toTypeArgument, i) and + fromTypeArgument = getTypeArgumentRanked(ugt, _, i) + } + + /** + * Holds if `fromTypeArgument` is identity convertible to `toTypeArgument` and + * both types are the `i`th type argument in _some_ constructed type + * based on unbound generic type `ugt`. + */ + pragma[noinline] + private predicate convTypeArgumentsSameUnbound(UnboundGenericType ugt, TypeArgument fromTypeArgument, TypeArgument toTypeArgument, int i) { + convTypeArgumentsSomeUnbound(ugt, fromTypeArgument, toTypeArgument, i) and + toTypeArgument = getTypeArgumentRanked(ugt, _, i) + } + + pragma[nomagic] + private predicate convIdentitySingle0(UnboundGenericType ugt, IdentityConvertibleConstructedType toType, TypeArgument fromTypeArgument, TypeArgument toTypeArgument) { + convTypeArgumentsSameUnbound(ugt, fromTypeArgument, toTypeArgument, 0) and + toTypeArgument = getTypeArgumentRanked(ugt, toType, 0) and + ugt.getNumberOfTypeParameters() = 1 + } + + /** + * Holds if the type arguments of types `fromType` and `toType` are identity + * convertible, and the number of type arguments is 1. + */ + predicate convIdentitySingle(UnboundGenericType ugt, IdentityConvertibleConstructedType fromType, IdentityConvertibleConstructedType toType) { + exists(TypeArgument fromTypeArgument, TypeArgument toTypeArgument | + convIdentitySingle0(ugt, toType, fromTypeArgument, toTypeArgument) | + fromTypeArgument = getTypeArgumentRanked(ugt, fromType, 0) + ) + } + + pragma[nomagic] + private predicate convIdentityMultiple01Aux0(UnboundGenericType ugt, IdentityConvertibleConstructedType toType, TypeArgument fromTypeArgument0, TypeArgument toTypeArgument0, TypeArgument toTypeArgument1) { + convTypeArgumentsSameUnbound(ugt, fromTypeArgument0, toTypeArgument0, 0) and + toTypeArgument0 = getTypeArgumentRanked(ugt, toType, 0) and + toTypeArgument1 = getTypeArgumentRanked(ugt, toType, 1) + } + + pragma[nomagic] + private predicate convIdentityMultiple01Aux1(UnboundGenericType ugt, IdentityConvertibleConstructedType fromType, TypeArgument fromTypeArgument0, TypeArgument fromTypeArgument1, TypeArgument toTypeArgument1) { + convTypeArgumentsSameUnbound(ugt, fromTypeArgument1, toTypeArgument1, 1) and + fromTypeArgument0 = getTypeArgumentRanked(ugt, fromType, 0) and + fromTypeArgument1 = getTypeArgumentRanked(ugt, fromType, 1) + } + + /** + * Holds if the first two ranked type arguments of types `fromType` and `toType` + * are identity convertible. + */ + private predicate convIdentityMultiple01(UnboundGenericType ugt, IdentityConvertibleConstructedType fromType, IdentityConvertibleConstructedType toType) { + exists(Type fromTypeArgument0, Type toTypeArgument0, Type fromTypeArgument1, Type toTypeArgument1 | + convIdentityMultiple01Aux0(ugt, toType, fromTypeArgument0, toTypeArgument0, toTypeArgument1) | + convIdentityMultiple01Aux1(ugt, fromType, fromTypeArgument0, fromTypeArgument1, toTypeArgument1) + ) + } + + pragma[nomagic] + private predicate convIdentityMultiple2Aux(UnboundGenericType ugt, IdentityConvertibleConstructedType toType, int i, TypeArgument fromTypeArgument, TypeArgument toTypeArgument) { + convTypeArgumentsSameUnbound(ugt, fromTypeArgument, toTypeArgument, i) and + toTypeArgument = getTypeArgumentRanked(ugt, toType, i) and + i >= 2 + } + + private predicate convIdentityMultiple2(UnboundGenericType ugt, IdentityConvertibleConstructedType fromType, IdentityConvertibleConstructedType toType, int i) { + exists(TypeArgument fromTypeArgument, TypeArgument toTypeArgument | + convIdentityMultiple2Aux(ugt, toType, i, fromTypeArgument, toTypeArgument) | + fromTypeArgument = getTypeArgumentRanked(ugt, fromType, i) + ) + } + + /** + * Holds if the ranked type arguments 0 through `i` (with `i >= 1`) of types + * `fromType` and `toType` are identity convertible. + */ + pragma[nomagic] + predicate convIdentityMultiple(UnboundGenericType ugt, IdentityConvertibleConstructedType fromType, IdentityConvertibleConstructedType toType, int i) { + convIdentityMultiple01(ugt, fromType, toType) and i = 1 + or + convIdentityMultiple(ugt, fromType, toType, i - 1) and + convIdentityMultiple2(ugt, fromType, toType, i) + } + + private predicate convIdentityStrictConstructedType(IdentityConvertibleConstructedType fromType, IdentityConvertibleConstructedType toType) { + /* Semantically equivalent with + * ``` + * ugt = fromType.getUnboundGeneric() + * and + * forex(int i | + * i in [0 .. ugt.getNumberOfTypeParameters() - 1] | + * exists(Type t1, Type t2 | + * t1 = getTypeArgument(ugt, fromType, i, _) and + * t2 = getTypeArgument(ugt, toType, i, _) | + * convIdentity(t1, t2) + * ) + * ) + * ``` + * but performance is improved by explicitly evaluating the `i`th argument + * only when all preceding arguments are convertible. + */ + fromType != toType and + ( + convIdentitySingle(_, fromType, toType) + or + exists(UnboundGenericType ugt | + convIdentityMultiple(ugt, fromType, toType, ugt.getNumberOfTypeParameters() - 1) + ) + ) } } @@ -363,9 +465,8 @@ private predicate convRefTypeNonNull(Type fromType, Type toType) { * This is a deliberate, small cartesian product, so we have manually lifted it to force the * evaluator to evaluate it in its entirety, rather than trying to optimize it in context. */ -pragma [noinline] -private -predicate defaultDynamicConversion(Type fromType, Type toType) { +pragma[noinline] +private predicate defaultDynamicConversion(Type fromType, Type toType) { fromType instanceof RefType and toType instanceof DynamicType } @@ -373,9 +474,8 @@ predicate defaultDynamicConversion(Type fromType, Type toType) { * This is a deliberate, small cartesian product, so we have manually lifted it to force the * evaluator to evaluate it in its entirety, rather than trying to optimize it in context. */ -pragma [noinline] -private -predicate defaultDelegateConversion(RefType fromType, RefType toType) { +pragma[noinline] +private predicate defaultDelegateConversion(RefType fromType, RefType toType) { fromType instanceof DelegateType and toType = any(SystemDelegateClass c).getABaseType*() } @@ -398,9 +498,8 @@ private predicate convRefTypeRefType(RefType fromType, RefType toType) { * This is a deliberate, small cartesian product, so we have manually lifted it to force the * evaluator to evaluate it in its entirety, rather than trying to optimize it in context. */ -pragma [noinline] -private -predicate defaultArrayConversion(Type fromType, RefType toType) { +pragma[noinline] +private predicate defaultArrayConversion(Type fromType, RefType toType) { fromType instanceof ArrayType and toType = any(SystemArrayClass c).getABaseType*() } @@ -427,7 +526,7 @@ private predicate convArrayTypeCovariance(ArrayType fromType, ArrayType toType) convArrayTypeCovarianceJoin(fromType, toType, toType.getDimension(), toType.getRank()) } -pragma [noinline] +pragma[noinline] private predicate convArrayTypeCovarianceJoin(ArrayType fromType, ArrayType toType, int dim, int rnk) { convArrayElementType(fromType, toType.getElementType(), dim, rnk) } @@ -616,7 +715,7 @@ predicate convConversionOperator(Type fromType, Type toType) { } /** 13.1.3.2: Variance conversion. */ -private predicate convVariance(VarianceConvertibleConstructedType fromType, VarianceConvertibleConstructedType toType) { +private predicate convVariance(ConstructedType fromType, ConstructedType toType) { /* Semantically equivalent with * ``` * ugt = fromType.getUnboundGeneric() @@ -637,95 +736,204 @@ private predicate convVariance(VarianceConvertibleConstructedType fromType, Vari * but performance is improved by explicitly evaluating the `i`th argument * only when all preceding arguments are convertible. */ - exists(UnboundGenericType ugt | - convVarianceFromZero(ugt, fromType, toType, ugt.getNumberOfTypeParameters() - 1) - ) -} - -/** - * Holds if the type arguments 0 through `i` of types `fromType` and `toType` - * are variance convertible. - */ -private predicate convVarianceFromZero(UnboundGenericType ugt, VarianceConvertibleConstructedType fromType, VarianceConvertibleConstructedType toType, int i) { - exists(Type toTypeArgument | - convVarianceFromZeroAux(ugt, fromType, i, toTypeArgument) | - toTypeArgument = getTypeArgument(ugt, toType, i, _) and - i = 0 and - fromType != toType - ) + Variance::convVarianceSingle(_, fromType, toType) or - exists(Type toTypeArgument | - convVarianceFromZero(ugt, fromType, toType, i - 1) | - toTypeArgument = getTypeArgument(ugt, toType, i, _) and - convVarianceFromZeroAux(ugt, fromType, i, toTypeArgument) + exists(UnboundGenericType ugt | + Variance::convVarianceMultiple(ugt, fromType, toType, ugt.getNumberOfTypeParameters() - 1) ) } -pragma [nomagic] -private predicate convVarianceFromZeroAux(UnboundGenericType ugt, VarianceConvertibleConstructedType fromType, int i, Type toTypeArgument) { - exists(Type fromTypeArgument, TypeParameter tp | - fromTypeArgument = getTypeArgument(ugt, fromType, i, tp) | - convIdentity(fromTypeArgument, toTypeArgument) +private module Variance { + /** + * Holds if constructed type `ct` is potentially variance convertible to + * or from another constructed type, as a result of the `i`th type + * argument being potentially convertible. + */ + private predicate isVarianceConvertible(ConstructedType ct, int i) { + exists(TypeParameter tp, Type t | + tp = ct.getUnboundGeneric().getTypeParameter(i) and + t = ct.getTypeArgument(i) | + ( + // Anything that is not a type parameter is potentially convertible + // to/from another type; if the `i`th type parameter is invariant, + // `t` must be strictly identity convertible + not t instanceof TypeParameter + and + (tp.isIn() or tp.isOut() or Identity::convIdentityStrict(t, _)) + ) + or + exists(TypeParameter s | + s = t | + // A type parameter with implicit reference conversion + exists(convTypeParameterBase(s)) and s.isRefType() and tp.isOut() + or + // A type parameter convertible from another type parameter + exists(TypeParameter u | s = convTypeParameterBase(u) and u.isRefType() and tp.isIn()) + ) + ) + } + + private class VarianceConvertibleConstructedType extends ConstructedType { + VarianceConvertibleConstructedType() { + isVarianceConvertible(this, _) + } + } + + /** + * Gets the number of different type arguments supplied for the type + * parameter at index `i` in unbound generic type `ugt`. + */ + private int getTypeArgumentCount(UnboundGenericType ugt, int i) { + result = strictcount(Type arg | + exists(VarianceConvertibleConstructedType ct | + ct.getUnboundGeneric() = ugt | + arg = ct.getTypeArgument(i) + ) + ) + } + + private int rnk(UnboundGenericType ugt, int i) { + result = rank[i + 1](int j, int k | + j = getTypeArgumentCount(ugt, k) | + k order by j, k + ) + } + + private Type getTypeArgumentRanked(UnboundGenericType ugt, VarianceConvertibleConstructedType t, int i, TypeParameter tp) { + result = getTypeArgument(ugt, t, rnk(ugt, i), tp) + } + + pragma[noinline] + private Type getATypeArgumentRankedOut(int i) { + result = getTypeArgumentRanked(_, _, i, any(TypeParameter tp | tp.isOut())) + } + + pragma[noinline] + private Type getATypeArgumentRankedIn(int i) { + result = getTypeArgumentRanked(_, _, i, any(TypeParameter tp | tp.isIn())) + } + + private predicate convRefTypeTypeArgumentOut(TypeArgument fromType, TypeArgument toType, int i) { + convRefTypeNonNull(fromType, toType) and + toType = getATypeArgumentRankedOut(i) + } + + private predicate convRefTypeTypeArgumentIn(TypeArgument toType, TypeArgument fromType, int i) { + convRefTypeNonNull(toType, fromType) and + toType = getATypeArgumentRankedIn(i) + } + + private newtype TVariance = TNone() or TIn() or TOut() + + /** + * Holds if `fromTypeArgument` is convertible to `toTypeArgument`, with variance + * `v`, and both types are the `i`th type argument in _some_ constructed type. + */ + pragma[nomagic] + private predicate convTypeArguments(TypeArgument fromTypeArgument, TypeArgument toTypeArgument, int i, TVariance v) { + exists(int j | + fromTypeArgument = getTypeArgumentRanked(_, _, i, _) and + toTypeArgument = getTypeArgumentRanked(_, _, j, _) and + i <= j and j <= i + | + convIdentity(fromTypeArgument, toTypeArgument) and + v = TNone() + or + convRefTypeTypeArgumentOut(fromTypeArgument, toTypeArgument, j) and + v = TOut() + or + convRefTypeTypeArgumentIn(toTypeArgument, fromTypeArgument, j) and + v = TIn() + ) + } + + pragma[nomagic] + private predicate convTypeArgumentsSomeUnbound(UnboundGenericType ugt, TypeArgument fromTypeArgument, TypeArgument toTypeArgument, int i) { + exists(TypeParameter tp, TVariance v | + convTypeArguments(fromTypeArgument, toTypeArgument, i, v) | + fromTypeArgument = getTypeArgumentRanked(ugt, _, i, tp) and + (v = TIn() implies tp.isIn()) and + (v = TOut() implies tp.isOut()) + ) + } + + /** + * Holds if `fromTypeArgument` is convertible to `toTypeArgument` and + * both types are the `i`th type argument in _some_ constructed type + * based on unbound generic type `ugt`. + */ + pragma[noinline] + private predicate convTypeArgumentsSameUnbound(UnboundGenericType ugt, TypeArgument fromTypeArgument, TypeArgument toTypeArgument, int i) { + convTypeArgumentsSomeUnbound(ugt, fromTypeArgument, toTypeArgument, i) and + toTypeArgument = getTypeArgumentRanked(ugt, _, i, _) + } + + pragma[nomagic] + private predicate convVarianceSingle0(UnboundGenericType ugt, VarianceConvertibleConstructedType toType, TypeArgument fromTypeArgument, TypeArgument toTypeArgument) { + convTypeArgumentsSameUnbound(ugt, fromTypeArgument, toTypeArgument, 0) and + toTypeArgument = getTypeArgumentRanked(ugt, toType, 0, _) and + ugt.getNumberOfTypeParameters() = 1 + } + + /** + * Holds if the type arguments of types `fromType` and `toType` are variance + * convertible, and the number of type arguments is 1. + */ + predicate convVarianceSingle(UnboundGenericType ugt, VarianceConvertibleConstructedType fromType, VarianceConvertibleConstructedType toType) { + exists(TypeArgument fromTypeArgument, TypeArgument toTypeArgument | + convVarianceSingle0(ugt, toType, fromTypeArgument, toTypeArgument) | + fromTypeArgument = getTypeArgumentRanked(ugt, fromType, 0, _) + ) + } + + pragma[nomagic] + private predicate convVarianceMultiple01Aux0(UnboundGenericType ugt, VarianceConvertibleConstructedType toType, TypeArgument fromTypeArgument0, TypeArgument toTypeArgument0, TypeArgument toTypeArgument1) { + convTypeArgumentsSameUnbound(ugt, fromTypeArgument0, toTypeArgument0, 0) and + toTypeArgument0 = getTypeArgumentRanked(ugt, toType, 0, _) and + toTypeArgument1 = getTypeArgumentRanked(ugt, toType, 1, _) + } + + pragma[nomagic] + private predicate convVarianceMultiple01Aux1(UnboundGenericType ugt, VarianceConvertibleConstructedType fromType, TypeArgument fromTypeArgument0, TypeArgument fromTypeArgument1, TypeArgument toTypeArgument1) { + convTypeArgumentsSameUnbound(ugt, fromTypeArgument1, toTypeArgument1, 1) and + fromTypeArgument0 = getTypeArgumentRanked(ugt, fromType, 0, _) and + fromTypeArgument1 = getTypeArgumentRanked(ugt, fromType, 1, _) + } + + /** + * Holds if the first two ranked type arguments of types `fromType` and `toType` + * are variance convertible. + */ + private predicate convVarianceMultiple01(UnboundGenericType ugt, VarianceConvertibleConstructedType fromType, VarianceConvertibleConstructedType toType) { + exists(TypeArgument fromTypeArgument0, TypeArgument toTypeArgument0, TypeArgument fromTypeArgument1, TypeArgument toTypeArgument1 | + convVarianceMultiple01Aux0(ugt, toType, fromTypeArgument0, toTypeArgument0, toTypeArgument1) | + convVarianceMultiple01Aux1(ugt, fromType, fromTypeArgument0, fromTypeArgument1, toTypeArgument1) + ) + } + + pragma[nomagic] + private predicate convVarianceMultiple2Aux(UnboundGenericType ugt, VarianceConvertibleConstructedType toType, int i, TypeArgument fromTypeArgument, TypeArgument toTypeArgument) { + convTypeArgumentsSameUnbound(ugt, fromTypeArgument, toTypeArgument, i) and + toTypeArgument = getTypeArgumentRanked(ugt, toType, i, _) and + i >= 2 + } + + private predicate convVarianceMultiple2(UnboundGenericType ugt, VarianceConvertibleConstructedType fromType, VarianceConvertibleConstructedType toType, int i) { + exists(TypeArgument fromTypeArgument, TypeArgument toTypeArgument | + convVarianceMultiple2Aux(ugt, toType, i, fromTypeArgument, toTypeArgument) | + fromTypeArgument = getTypeArgumentRanked(ugt, fromType, i, _) + ) + } + + /** + * Holds if the ranked type arguments 0 through `i` (with `i >= 1`) of types + * `fromType` and `toType` are variance convertible. + */ + pragma[nomagic] + predicate convVarianceMultiple(UnboundGenericType ugt, VarianceConvertibleConstructedType fromType, VarianceConvertibleConstructedType toType, int i) { + convVarianceMultiple01(ugt, fromType, toType) and i = 1 or - convRefTypeTypeArgumentOut(fromTypeArgument, toTypeArgument, i) and - tp.isOut() - or - convRefTypeTypeArgumentIn(toTypeArgument, fromTypeArgument, i) and - tp.isIn() - ) -} - -pragma [nomagic] -private predicate convRefTypeTypeArgumentOut(TypeArgument fromType, TypeArgument toType, int i) { - exists(TypeParameter tp | - convRefTypeNonNull(fromType, toType) | - toType = getTypeArgument(_, _, i, tp) and - tp.isOut() - ) -} - -pragma [nomagic] -private predicate convRefTypeTypeArgumentIn(TypeArgument toType, TypeArgument fromType, int i) { - exists(TypeParameter tp | - convRefTypeNonNull(toType, fromType) | - toType = getTypeArgument(_, _, i, tp) and - tp.isIn() - ) -} - -private class VarianceConvertibleConstructedType extends ConstructedType { - VarianceConvertibleConstructedType() { - isVarianceConvertible(this, _) + convVarianceMultiple(ugt, fromType, toType, i - 1) and + convVarianceMultiple2(ugt, fromType, toType, i) } } - -/** - * Holds if constructed type `ct` is potentially variance convertible to - * or from another constructed type, as a result of the `i`th type - * argument being potentially convertible. - */ -private predicate isVarianceConvertible(ConstructedType ct, int i) { - exists(TypeParameter tp, Type t | - tp = ct.getUnboundGeneric().getTypeParameter(i) - and - t = ct.getTypeArgument(i) - | - ( - // Anything that is not a type parameter is potentially convertible - // to/from another type; if the `i`th type parameter is invariant, - // `t` must be strictly identity convertible - not t instanceof TypeParameter - and - (tp.isIn() or tp.isOut() or convIdentityStrict(t, _)) - ) - or - exists(TypeParameter s | - s = t | - // A type parameter with implicit reference conversion - exists(convTypeParameterBase(s)) and s.isRefType() and tp.isOut() - or - // A type parameter convertible from another type parameter - exists(TypeParameter u | s = convTypeParameterBase(u) and u.isRefType() and tp.isIn()) - ) - ) -} From f706d2a96cd41b493a0c185df794146b8d57e745 Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Sun, 7 Oct 2018 17:29:38 +0100 Subject: [PATCH 13/58] CPP: Change notes. --- change-notes/1.19/analysis-cpp.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/change-notes/1.19/analysis-cpp.md b/change-notes/1.19/analysis-cpp.md index b1f84b60e51..9fd5a6d86fb 100644 --- a/change-notes/1.19/analysis-cpp.md +++ b/change-notes/1.19/analysis-cpp.md @@ -14,6 +14,8 @@ |----------------------------|------------------------|------------------------------------------------------------------| | Resource not released in destructor | Fewer false positive results | Placement new is now excluded from the query. | | Missing return statement (`cpp/missing-return`) | Visible by default | The precision of this query has been increased from 'medium' to 'high', which makes it visible by default in LGTM. It was 'medium' in release 1.17 and 1.18 because it had false positives due to an extractor bug that was fixed in 1.18. | +| Call to memory access function may overflow buffer | More correct results | Array indexing with a negative index is now detected by this query. | +| Suspicious add with sizeof | Fewer false positive results | Arithmetic with void pointers (where allowed) is now excluded from this query. | | Wrong type of arguments to formatting function | Fewer false positive results | False positive results involving typedefs have been removed. Expected argument types are determined more accurately, especially for wide string and pointer types. Custom (non-standard) formatting functions are also identified more accurately. | ## Changes to QL libraries From 4fb6611dbe69547d8b2946b7d6dc96cbe9c7af6b Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Mon, 8 Oct 2018 12:18:37 +0100 Subject: [PATCH 14/58] CPP: Change note for #264. --- change-notes/1.19/analysis-cpp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/change-notes/1.19/analysis-cpp.md b/change-notes/1.19/analysis-cpp.md index 9fd5a6d86fb..b3905711c41 100644 --- a/change-notes/1.19/analysis-cpp.md +++ b/change-notes/1.19/analysis-cpp.md @@ -6,7 +6,7 @@ | **Query** | **Tags** | **Purpose** | |-----------------------------|-----------|--------------------------------------------------------------------| -| *@name of query (Query ID)* | *Tags* |*Aim of the new query and whether it is enabled by default or not* | +| Cast from char* to wchar_t* | security, external/cwe/cwe-704, external/microsoft/c/c6276 | Detects potentially dangerous casts from char* to wchar_t*. Enabled by default on LGTM. | ## Changes to existing queries From ff2abe035630cf287d5e7bd1ee9044376e196ad6 Mon Sep 17 00:00:00 2001 From: Tom Hvitved Date: Mon, 8 Oct 2018 13:33:50 +0200 Subject: [PATCH 15/58] C#: Add qldoc to `getTypeArgumentRanked()` --- csharp/ql/src/semmle/code/csharp/Conversion.qll | 2 ++ 1 file changed, 2 insertions(+) diff --git a/csharp/ql/src/semmle/code/csharp/Conversion.qll b/csharp/ql/src/semmle/code/csharp/Conversion.qll index bb4c697d7dc..a9709221dcc 100644 --- a/csharp/ql/src/semmle/code/csharp/Conversion.qll +++ b/csharp/ql/src/semmle/code/csharp/Conversion.qll @@ -169,6 +169,7 @@ private module Identity { ) } + /** Gets the 'i'th type argument, ranked by size, of constructed type `t`. */ private Type getTypeArgumentRanked(UnboundGenericType ugt, IdentityConvertibleConstructedType t, int i) { result = getTypeArgument(ugt, t, rnk(ugt, i), _) } @@ -799,6 +800,7 @@ private module Variance { ) } + /** Gets the 'i'th type argument, ranked by size, of constructed type `t`. */ private Type getTypeArgumentRanked(UnboundGenericType ugt, VarianceConvertibleConstructedType t, int i, TypeParameter tp) { result = getTypeArgument(ugt, t, rnk(ugt, i), tp) } From 70cd03d3bcd47f89bd9ba0dcb6a18598c4dfb3d8 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Mon, 8 Oct 2018 15:47:11 +0200 Subject: [PATCH 16/58] JS: use DataFlow::ArrayCreationNode in additional places --- javascript/ql/src/AngularJS/InsecureUrlWhitelist.ql | 4 ++-- .../ql/src/Expressions/UnboundEventHandlerReceiver.ql | 2 +- .../src/semmle/javascript/dataflow/Configuration.qll | 2 +- .../frameworks/AngularJS/DependencyInjections.qll | 2 +- .../ql/src/semmle/javascript/frameworks/Express.qll | 2 +- .../semmle/javascript/frameworks/ExpressModules.qll | 10 +++++----- .../javascript/security/dataflow/CommandInjection.qll | 4 ++-- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/javascript/ql/src/AngularJS/InsecureUrlWhitelist.ql b/javascript/ql/src/AngularJS/InsecureUrlWhitelist.ql index 9ab3ff64f58..06371bdc3a7 100644 --- a/javascript/ql/src/AngularJS/InsecureUrlWhitelist.ql +++ b/javascript/ql/src/AngularJS/InsecureUrlWhitelist.ql @@ -17,7 +17,7 @@ import javascript * Holds if `setupCall` is a call to `$sceDelegateProvider.resourceUrlWhitelist` with * argument `list`. */ -predicate isResourceUrlWhitelist(DataFlow::MethodCallNode setupCall, DataFlow::ArrayLiteralNode list) { +predicate isResourceUrlWhitelist(DataFlow::MethodCallNode setupCall, DataFlow::ArrayCreationNode list) { exists (AngularJS::ServiceReference service | service.getName() = "$sceDelegateProvider" and setupCall.asExpr() = service.getAMethodCall("resourceUrlWhitelist") and @@ -33,7 +33,7 @@ class ResourceUrlWhitelistEntry extends Expr { string pattern; ResourceUrlWhitelistEntry() { - exists (DataFlow::ArrayLiteralNode whitelist | + exists (DataFlow::ArrayCreationNode whitelist | isResourceUrlWhitelist(setupCall, whitelist) and this = whitelist.getAnElement().asExpr() and this.mayHaveStringValue(pattern) diff --git a/javascript/ql/src/Expressions/UnboundEventHandlerReceiver.ql b/javascript/ql/src/Expressions/UnboundEventHandlerReceiver.ql index de947b250ab..347ac26afa1 100644 --- a/javascript/ql/src/Expressions/UnboundEventHandlerReceiver.ql +++ b/javascript/ql/src/Expressions/UnboundEventHandlerReceiver.ql @@ -36,7 +36,7 @@ private predicate isBoundInMethod(MethodDeclaration method) { bindAll.getArgument(1).mayHaveStringValue(name) or // _.bindAll(this, [, ]) - exists (DataFlow::ArrayLiteralNode names | + exists (DataFlow::ArrayCreationNode names | names.flowsTo(bindAll.getArgument(1)) and names.getAnElement().mayHaveStringValue(name) ) diff --git a/javascript/ql/src/semmle/javascript/dataflow/Configuration.qll b/javascript/ql/src/semmle/javascript/dataflow/Configuration.qll index 6a2abe2e04e..57a2aef34ed 100644 --- a/javascript/ql/src/semmle/javascript/dataflow/Configuration.qll +++ b/javascript/ql/src/semmle/javascript/dataflow/Configuration.qll @@ -336,7 +336,7 @@ private class LibraryPartialCall extends AdditionalPartialInvokeNode { override predicate isPartialArgument(DataFlow::Node callback, DataFlow::Node argument, int index) { callback = getArgument(0) and - exists (DataFlow::ArrayLiteralNode array | + exists (DataFlow::ArrayCreationNode array | array.flowsTo(getArgument(1)) and argument = array.getElement(index)) } diff --git a/javascript/ql/src/semmle/javascript/frameworks/AngularJS/DependencyInjections.qll b/javascript/ql/src/semmle/javascript/frameworks/AngularJS/DependencyInjections.qll index 5be58e731ec..ea4a7599ef2 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/AngularJS/DependencyInjections.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/AngularJS/DependencyInjections.qll @@ -146,7 +146,7 @@ private DataFlow::PropWrite getAPropertyDependencyInjection(Function function) { */ private class FunctionWithInjectProperty extends InjectableFunction { override Function astNode; - DataFlow::ArrayLiteralNode dependencies; + DataFlow::ArrayCreationNode dependencies; FunctionWithInjectProperty() { ( diff --git a/javascript/ql/src/semmle/javascript/frameworks/Express.qll b/javascript/ql/src/semmle/javascript/frameworks/Express.qll index afd6108c54c..484ffaf0c49 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/Express.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/Express.qll @@ -891,7 +891,7 @@ module Express { getMethodName() = methodName and exists (DataFlow::ValueNode arg | arg = getAnArgument() | - exists (DataFlow::ArrayLiteralNode array | + exists (DataFlow::ArrayCreationNode array | array.flowsTo(arg) and routeHandlerArg = array.getAnElement() ) or diff --git a/javascript/ql/src/semmle/javascript/frameworks/ExpressModules.qll b/javascript/ql/src/semmle/javascript/frameworks/ExpressModules.qll index 03d8d92ea0e..e7538c54612 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/ExpressModules.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/ExpressModules.qll @@ -144,8 +144,8 @@ module ExpressLibraries { override DataFlow::Node getASecretKey() { exists (DataFlow::Node secret | secret = getOption("secret") | - if exists(DataFlow::ArrayLiteralNode arr | arr.flowsTo(secret)) then - result = any (DataFlow::ArrayLiteralNode arr | arr.flowsTo(secret)).getAnElement() + if exists(DataFlow::ArrayCreationNode arr | arr.flowsTo(secret)) then + result = any (DataFlow::ArrayCreationNode arr | arr.flowsTo(secret)).getAnElement() else result = secret ) @@ -182,8 +182,8 @@ module ExpressLibraries { override DataFlow::Node getASecretKey() { exists (DataFlow::Node arg0 | arg0 = getArgument(0) | - if exists(DataFlow::ArrayLiteralNode arr | arr.flowsTo(arg0)) then - result = any (DataFlow::ArrayLiteralNode arr | arr.flowsTo(arg0)).getAnElement() + if exists(DataFlow::ArrayCreationNode arr | arr.flowsTo(arg0)) then + result = any (DataFlow::ArrayCreationNode arr | arr.flowsTo(arg0)).getAnElement() else result = arg0 ) @@ -220,7 +220,7 @@ module ExpressLibraries { override DataFlow::Node getASecretKey() { result = getOption("secret") or - exists (DataFlow::ArrayLiteralNode keys | + exists (DataFlow::ArrayCreationNode keys | keys.flowsTo(getOption("keys")) and result = keys.getAnElement() ) diff --git a/javascript/ql/src/semmle/javascript/security/dataflow/CommandInjection.qll b/javascript/ql/src/semmle/javascript/security/dataflow/CommandInjection.qll index facc320973f..0e817900802 100644 --- a/javascript/ql/src/semmle/javascript/security/dataflow/CommandInjection.qll +++ b/javascript/ql/src/semmle/javascript/security/dataflow/CommandInjection.qll @@ -75,7 +75,7 @@ module CommandInjection { ArgumentListTracking() { this = "ArgumentListTracking" } override predicate isSource(DataFlow::Node nd) { - nd instanceof DataFlow::ArrayLiteralNode + nd instanceof DataFlow::ArrayCreationNode or exists (StringLiteral shell | shellCmd(shell, _) | nd = DataFlow::valueNode(shell) @@ -125,7 +125,7 @@ module CommandInjection { * we want to report the `spawn` call as the sink, so we bind it to `sys`. */ private predicate indirectCommandInjection(DataFlow::Node sink, SystemCommandExecution sys) { - exists (ArgumentListTracking cfg, DataFlow::ArrayLiteralNode args, + exists (ArgumentListTracking cfg, DataFlow::ArrayCreationNode args, StringLiteral shell, string dashC | shellCmd(shell, dashC) and cfg.hasFlow(DataFlow::valueNode(shell), sys.getACommandArgument()) and From 03fd1ce83d655e04dcdb0290c032a4c61a8e6ed4 Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Mon, 8 Oct 2018 15:30:43 +0100 Subject: [PATCH 17/58] CPP: Remove external/microsoft tag. --- change-notes/1.19/analysis-cpp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/change-notes/1.19/analysis-cpp.md b/change-notes/1.19/analysis-cpp.md index b3905711c41..39c5941098d 100644 --- a/change-notes/1.19/analysis-cpp.md +++ b/change-notes/1.19/analysis-cpp.md @@ -6,7 +6,7 @@ | **Query** | **Tags** | **Purpose** | |-----------------------------|-----------|--------------------------------------------------------------------| -| Cast from char* to wchar_t* | security, external/cwe/cwe-704, external/microsoft/c/c6276 | Detects potentially dangerous casts from char* to wchar_t*. Enabled by default on LGTM. | +| Cast from char* to wchar_t* | security, external/cwe/cwe-704 | Detects potentially dangerous casts from char* to wchar_t*. Enabled by default on LGTM. | ## Changes to existing queries From c747f24b39e6a95db82db2fbf390efe9079fbd6a Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Mon, 1 Oct 2018 17:33:18 +0100 Subject: [PATCH 18/58] CPP: Fix the initialized array case in getBufferSize. --- cpp/ql/src/semmle/code/cpp/commons/Buffer.qll | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll b/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll index b184832e166..0ceee2b7d08 100644 --- a/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll +++ b/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll @@ -58,6 +58,10 @@ int getBufferSize(Expr bufferExpr, Element why) { // buffer is an initialized array // e.g. int buffer[] = {1, 2, 3}; why = bufferVar.getInitializer().getExpr() and + ( + why instanceof AggregateLiteral or + why instanceof StringLiteral + ) and result = why.(Expr).getType().(ArrayType).getSize() and not exists(bufferVar.getType().getUnspecifiedType().(ArrayType).getSize()) ) or exists(Class parentClass, VariableAccess parentPtr | From ef8ca5de587e79e6435f68da10088871998de0c2 Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Tue, 2 Oct 2018 09:15:35 +0100 Subject: [PATCH 19/58] CPP: Replace def-use with dataflow in getBufferSize. --- cpp/ql/src/semmle/code/cpp/commons/Buffer.qll | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll b/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll index 0ceee2b7d08..1584ec8a56e 100644 --- a/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll +++ b/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll @@ -1,4 +1,5 @@ import cpp +import semmle.code.cpp.dataflow.DataFlow /** * Holds if `sizeof(s)` occurs as part of the parameter of a dynamic @@ -75,16 +76,13 @@ int getBufferSize(Expr bufferExpr, Element why) { bufferVar.getType().getSize() - parentClass.getSize() ) - ) or exists(Expr def | + ) or ( // buffer is assigned with an allocation - definitionUsePair(_, def, bufferExpr) and - exprDefinition(_, def, why) and + DataFlow::localFlowStep(DataFlow::exprNode(why), DataFlow::exprNode(bufferExpr)) and isFixedSizeAllocationExpr(why, result) - ) or exists(Expr def, Expr e, Element why2 | - // buffer is assigned with another buffer - definitionUsePair(_, def, bufferExpr) and - exprDefinition(_, def, e) and - result = getBufferSize(e, why2) and + ) or exists(Expr def, Element why2 | + DataFlow::localFlowStep(DataFlow::exprNode(def), DataFlow::exprNode(bufferExpr)) and + result = getBufferSize(def, why2) and ( why = def or why = why2 From beb21f92d3b0c1506800e36522753243e0d8af81 Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Tue, 2 Oct 2018 10:20:46 +0100 Subject: [PATCH 20/58] CPP: Separate the dataflow case from dynamic allocation. --- cpp/ql/src/semmle/code/cpp/commons/Buffer.qll | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll b/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll index 1584ec8a56e..65aa7ebee20 100644 --- a/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll +++ b/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll @@ -77,10 +77,11 @@ int getBufferSize(Expr bufferExpr, Element why) { parentClass.getSize() ) ) or ( - // buffer is assigned with an allocation - DataFlow::localFlowStep(DataFlow::exprNode(why), DataFlow::exprNode(bufferExpr)) and - isFixedSizeAllocationExpr(why, result) + // buffer is a fixed size dynamic allocation + isFixedSizeAllocationExpr(bufferExpr, result) and + why = bufferExpr ) or exists(Expr def, Element why2 | + // dataflow DataFlow::localFlowStep(DataFlow::exprNode(def), DataFlow::exprNode(bufferExpr)) and result = getBufferSize(def, why2) and ( From fe6c9f9ea2b0bb3ece895496b6f6e9b591e2c774 Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Tue, 2 Oct 2018 10:27:40 +0100 Subject: [PATCH 21/58] CPP: Stricter dataflow in getBufferSize. --- cpp/ql/src/semmle/code/cpp/commons/Buffer.qll | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll b/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll index 65aa7ebee20..c27183360f7 100644 --- a/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll +++ b/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll @@ -80,13 +80,15 @@ int getBufferSize(Expr bufferExpr, Element why) { // buffer is a fixed size dynamic allocation isFixedSizeAllocationExpr(bufferExpr, result) and why = bufferExpr - ) or exists(Expr def, Element why2 | + ) or forex(Expr def | // dataflow - DataFlow::localFlowStep(DataFlow::exprNode(def), DataFlow::exprNode(bufferExpr)) and - result = getBufferSize(def, why2) and - ( - why = def or - why = why2 + DataFlow::localFlowStep(DataFlow::exprNode(def), DataFlow::exprNode(bufferExpr)) | + exists(Element why2 | + result = getBufferSize(def, why2) and + ( + why = def or + why = why2 + ) ) ) or exists(Type bufferType | // buffer is the address of a variable From 8ab830f21c371147fdb49bba876b7b21c6106b55 Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Tue, 2 Oct 2018 11:00:26 +0100 Subject: [PATCH 22/58] CPP: Allow multiple dataflow sources. --- cpp/ql/src/semmle/code/cpp/commons/Buffer.qll | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll b/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll index c27183360f7..6ef6c5f2832 100644 --- a/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll +++ b/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll @@ -80,15 +80,18 @@ int getBufferSize(Expr bufferExpr, Element why) { // buffer is a fixed size dynamic allocation isFixedSizeAllocationExpr(bufferExpr, result) and why = bufferExpr - ) or forex(Expr def | - // dataflow - DataFlow::localFlowStep(DataFlow::exprNode(def), DataFlow::exprNode(bufferExpr)) | - exists(Element why2 | - result = getBufferSize(def, why2) and - ( - why = def or - why = why2 - ) + ) or ( + // dataflow (all sources must be the same size) + forex(Expr def | + DataFlow::localFlowStep(DataFlow::exprNode(def), DataFlow::exprNode(bufferExpr)) | + result = getBufferSize(def, _) + ) and + + // find reason + exists(Expr def | + DataFlow::localFlowStep(DataFlow::exprNode(def), DataFlow::exprNode(bufferExpr)) | + why = def or + result = getBufferSize(def, why) ) ) or exists(Type bufferType | // buffer is the address of a variable From 8163def3ae29889a76cfff3e3fb53a74f361ad7f Mon Sep 17 00:00:00 2001 From: Geoffrey White <40627776+geoffw0@users.noreply.github.com> Date: Fri, 5 Oct 2018 18:37:05 +0100 Subject: [PATCH 23/58] CPP: Alter the dataflow case. --- cpp/ql/src/semmle/code/cpp/commons/Buffer.qll | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll b/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll index 6ef6c5f2832..a3de2f3eb59 100644 --- a/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll +++ b/cpp/ql/src/semmle/code/cpp/commons/Buffer.qll @@ -48,6 +48,7 @@ predicate memberMayBeVarSize(Class c, MemberVariable v) { /** * Get the size in bytes of the buffer pointed to by an expression (if this can be determined). */ +language[monotonicAggregates] int getBufferSize(Expr bufferExpr, Element why) { exists(Variable bufferVar | bufferVar = bufferExpr.(VariableAccess).getTarget() | ( @@ -82,16 +83,19 @@ int getBufferSize(Expr bufferExpr, Element why) { why = bufferExpr ) or ( // dataflow (all sources must be the same size) - forex(Expr def | + result = min(Expr def | DataFlow::localFlowStep(DataFlow::exprNode(def), DataFlow::exprNode(bufferExpr)) | - result = getBufferSize(def, _) + getBufferSize(def, _) + ) and result = max(Expr def | + DataFlow::localFlowStep(DataFlow::exprNode(def), DataFlow::exprNode(bufferExpr)) | + getBufferSize(def, _) ) and // find reason exists(Expr def | DataFlow::localFlowStep(DataFlow::exprNode(def), DataFlow::exprNode(bufferExpr)) | why = def or - result = getBufferSize(def, why) + exists(getBufferSize(def, why)) ) ) or exists(Type bufferType | // buffer is the address of a variable From 2fdf7667508adefc410b6998c9bf22b12dd81ce4 Mon Sep 17 00:00:00 2001 From: calum Date: Mon, 8 Oct 2018 17:26:30 +0100 Subject: [PATCH 24/58] C#: Address review comments. --- csharp/ql/src/Stubs/MinimalStubsFromSource.ql | 6 +---- csharp/ql/src/Stubs/Stubs.qll | 9 +++---- csharp/ql/src/Stubs/make_stubs.py | 26 +++++++++---------- 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/csharp/ql/src/Stubs/MinimalStubsFromSource.ql b/csharp/ql/src/Stubs/MinimalStubsFromSource.ql index 3a74abf8ae4..47ce45e467d 100644 --- a/csharp/ql/src/Stubs/MinimalStubsFromSource.ql +++ b/csharp/ql/src/Stubs/MinimalStubsFromSource.ql @@ -17,14 +17,10 @@ class UsedInSource extends GeneratedDeclaration { or this = any(Call c).getTarget() or - exists(ValueOrRefType t | t.fromSource() | this = t.getABaseType()) - or - exists(Variable v | v.fromSource() | this = v.getType()) + this = any(TypeMention tm).getType() or exists(Virtualizable v | v.fromSource() | this = v.getImplementee() or this = v.getOverridee()) or - this = any(Attribute a).getType() - or this = any(Attribute a).getType().getAConstructor() ) and diff --git a/csharp/ql/src/Stubs/Stubs.qll b/csharp/ql/src/Stubs/Stubs.qll index 4c9d67a9d4f..100cbdb1e56 100644 --- a/csharp/ql/src/Stubs/Stubs.qll +++ b/csharp/ql/src/Stubs/Stubs.qll @@ -239,7 +239,7 @@ private class GeneratedNamespace extends Namespace, GeneratedElement { final string getStubs() { result = getPreamble() + getTypeStubs() + - getSubNamespaces(0) + + getSubNamespaces() + getPostAmble() } @@ -254,10 +254,9 @@ private class GeneratedNamespace extends Namespace, GeneratedElement { result = count(GeneratedNamespace g | g.getParentNamespace() = this) } - private string getSubNamespaces(int n) { - result = getChildNamespace(n).getStubs() + getSubNamespaces(n+1) - or - n = getChildNamespaceCount() and result = "" + language[monotonicAggregates] + private string getSubNamespaces() { + result = concat(int i | exists(getChildNamespace(i)) | getChildNamespace(i).getStubs()) } private string getTypeStubs() { diff --git a/csharp/ql/src/Stubs/make_stubs.py b/csharp/ql/src/Stubs/make_stubs.py index 980da6da19f..e94cf0c7261 100644 --- a/csharp/ql/src/Stubs/make_stubs.py +++ b/csharp/ql/src/Stubs/make_stubs.py @@ -10,6 +10,7 @@ import sys import os +import subprocess print('Script to generate stub.cs files for C# qltest projects') @@ -42,9 +43,9 @@ if not foundCS: print("Test directory does not contain .cs files. Please specify a working qltest directory.") exit(1) -cmd = 'odasa selfTest' -print(cmd) -if os.system(cmd): +cmd = ['odasa', 'selfTest'] +print('Running ' + ' '.join(cmd)) +if subprocess.check_call(cmd): print("odasa selfTest failed. Ensure odasa is on your current path.") exit(1) @@ -57,9 +58,9 @@ if os.path.isfile(outputFile): os.remove(outputFile) # It would interfere with the test. print("Removed previous", outputFile) -cmd = 'odasa qltest --optimize --leave-temp-files "' + testDir + '"' -print(cmd) -if os.system(cmd) != 0: +cmd = ['odasa', 'qltest', '--optimize', '--leave-temp-files', testDir] +print('Running ' + ' '.join(cmd)) +if subprocess.check_call(cmd): print("qltest failed. Please fix up the test before proceeding.") exit(1) @@ -69,9 +70,9 @@ if not os.path.isdir(dbDir): print("Expected database directory " + dbDir + " not found. Please contact Semmle.") exit(1) -cmd = 'odasa runQuery --query "' + os.path.join(csharpQueries, 'MinimalStubsFromSource.ql') + '" --db "' + dbDir +'" --output-file "' + outputFile + '"' -print(cmd) -if os.system(cmd): +cmd = ['odasa', 'runQuery', '--query', os.path.join(csharpQueries, 'MinimalStubsFromSource.ql'), '--db', dbDir, '--output-file', outputFile] +print('Running ' + ' '.join(cmd)) +if subprocess.check_call(cmd): print('Failed to run the query to generate output file. Please contact Semmle.') exit(1) @@ -93,10 +94,9 @@ f = open(outputFile, "wb") f.write(contents) f.close() -cmd = 'odasa qltest --optimize "' + testDir + '"' -print(cmd) - -if os.system(cmd): +cmd = ['odasa', 'qltest', '--optimize', testDir] +print('Running ' + ' '.join(cmd)) +if subprocess.check_call(cmd): print('\nTest failed. You may need to fix up', outputFile) print('It may help to view', outputFile, ' in Visual Studio') print("Next steps:") From 7962530789d01cae0057fef7e12f29f15f007d38 Mon Sep 17 00:00:00 2001 From: yh-semmle Date: Wed, 19 Sep 2018 16:10:50 -0400 Subject: [PATCH 25/58] Java: add `.project` file in `test` directory --- java/ql/test/.project | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 java/ql/test/.project diff --git a/java/ql/test/.project b/java/ql/test/.project new file mode 100644 index 00000000000..169229c9cce --- /dev/null +++ b/java/ql/test/.project @@ -0,0 +1,12 @@ + + + semmlecode-tests-ql + + + + + + + com.semmle.plugin.qdt.core.qlnature + + From 3bc5e3bfdfa837081cab1bed2ed3666a843b146c Mon Sep 17 00:00:00 2001 From: Asger F Date: Wed, 3 Oct 2018 12:24:19 +0100 Subject: [PATCH 26/58] JS: Replace some uses AnalyzedValueNode with AnalyzedNode --- javascript/ql/src/semmle/javascript/StandardLibrary.qll | 6 +++--- .../dataflow/internal/InterProceduralTypeInference.qll | 4 ++-- .../src/semmle/javascript/frameworks/LodashUnderscore.qll | 6 +++--- javascript/ql/src/semmle/javascript/frameworks/React.qll | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/javascript/ql/src/semmle/javascript/StandardLibrary.qll b/javascript/ql/src/semmle/javascript/StandardLibrary.qll index e9d38d02c4f..f2dbaa4ef19 100644 --- a/javascript/ql/src/semmle/javascript/StandardLibrary.qll +++ b/javascript/ql/src/semmle/javascript/StandardLibrary.qll @@ -62,9 +62,9 @@ class JsonParseCall extends MethodCallExpr { * However, since the function could be invoked in another way, we additionally * still infer the ordinary abstract value. */ -private class AnalyzedThisInArrayIterationFunction extends AnalyzedValueNode, DataFlow::ThisNode { +private class AnalyzedThisInArrayIterationFunction extends AnalyzedNode, DataFlow::ThisNode { - AnalyzedValueNode thisSource; + AnalyzedNode thisSource; AnalyzedThisInArrayIterationFunction() { exists(DataFlow::MethodCallNode bindingCall, string name | @@ -82,7 +82,7 @@ private class AnalyzedThisInArrayIterationFunction extends AnalyzedValueNode, Da override AbstractValue getALocalValue() { result = thisSource.getALocalValue() or - result = AnalyzedValueNode.super.getALocalValue() + result = AnalyzedNode.super.getALocalValue() } } diff --git a/javascript/ql/src/semmle/javascript/dataflow/internal/InterProceduralTypeInference.qll b/javascript/ql/src/semmle/javascript/dataflow/internal/InterProceduralTypeInference.qll index e71561b4139..af4a928d6a1 100644 --- a/javascript/ql/src/semmle/javascript/dataflow/internal/InterProceduralTypeInference.qll +++ b/javascript/ql/src/semmle/javascript/dataflow/internal/InterProceduralTypeInference.qll @@ -10,7 +10,7 @@ import AbstractValuesImpl /** * Flow analysis for `this` expressions inside functions. */ -private abstract class AnalyzedThisExpr extends DataFlow::AnalyzedValueNode, DataFlow::ThisNode { +private abstract class AnalyzedThisExpr extends DataFlow::AnalyzedNode, DataFlow::ThisNode { DataFlow::FunctionNode binder; AnalyzedThisExpr() { @@ -29,7 +29,7 @@ private abstract class AnalyzedThisExpr extends DataFlow::AnalyzedValueNode, Dat */ private class AnalyzedThisInBoundFunction extends AnalyzedThisExpr { - AnalyzedValueNode thisSource; + AnalyzedNode thisSource; AnalyzedThisInBoundFunction() { exists(string name | diff --git a/javascript/ql/src/semmle/javascript/frameworks/LodashUnderscore.qll b/javascript/ql/src/semmle/javascript/frameworks/LodashUnderscore.qll index cdfe6035358..02a35a66793 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/LodashUnderscore.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/LodashUnderscore.qll @@ -27,9 +27,9 @@ module LodashUnderscore { * However, since the function could be invoked in another way, we additionally * still infer the ordinary abstract value. */ -private class AnalyzedThisInBoundCallback extends AnalyzedValueNode, DataFlow::ThisNode { +private class AnalyzedThisInBoundCallback extends AnalyzedNode, DataFlow::ThisNode { - AnalyzedValueNode thisSource; + AnalyzedNode thisSource; AnalyzedThisInBoundCallback() { exists(DataFlow::CallNode bindingCall, string binderName, int callbackIndex, int contextIndex, int argumentCount | @@ -128,7 +128,7 @@ private class AnalyzedThisInBoundCallback extends AnalyzedValueNode, DataFlow::T override AbstractValue getALocalValue() { result = thisSource.getALocalValue() or - result = AnalyzedValueNode.super.getALocalValue() + result = AnalyzedNode.super.getALocalValue() } } diff --git a/javascript/ql/src/semmle/javascript/frameworks/React.qll b/javascript/ql/src/semmle/javascript/frameworks/React.qll index a95d9e02c1a..316955ca77d 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/React.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/React.qll @@ -515,9 +515,9 @@ private class FactoryDefinition extends ReactElementDefinition { * However, since the function could be invoked in another way, we additionally * still infer the ordinary abstract value. */ -private class AnalyzedThisInBoundCallback extends AnalyzedValueNode, DataFlow::ThisNode { +private class AnalyzedThisInBoundCallback extends AnalyzedNode, DataFlow::ThisNode { - AnalyzedValueNode thisSource; + AnalyzedNode thisSource; AnalyzedThisInBoundCallback() { exists(DataFlow::CallNode bindingCall, string binderName | @@ -533,7 +533,7 @@ private class AnalyzedThisInBoundCallback extends AnalyzedValueNode, DataFlow::T override AbstractValue getALocalValue() { result = thisSource.getALocalValue() or - result = AnalyzedValueNode.super.getALocalValue() + result = AnalyzedNode.super.getALocalValue() } } From 030bae94546d52dd0066be2cf07dc66c43ca29dd Mon Sep 17 00:00:00 2001 From: Asger F Date: Wed, 3 Oct 2018 13:30:59 +0100 Subject: [PATCH 27/58] JS: Canonicalize ThisNode --- javascript/ql/src/semmle/javascript/Expr.qll | 10 +++++++ .../ql/src/semmle/javascript/Functions.qll | 11 ++++++++ .../semmle/javascript/dataflow/DataFlow.qll | 13 +++++++++ .../src/semmle/javascript/dataflow/Nodes.qll | 28 ++++++++++++++++--- .../semmle/javascript/dataflow/Sources.qll | 3 +- .../semmle/javascript/frameworks/React.qll | 4 +-- .../TaintTracking/BasicTaintTracking.expected | 2 ++ .../TaintTracking/thisAssignments.js | 10 +++++++ .../query-tests/Security/CWE-079/Xss.expected | 1 + .../Security/CWE-079/addEventListener.js | 3 ++ 10 files changed, 78 insertions(+), 7 deletions(-) create mode 100644 javascript/ql/test/library-tests/TaintTracking/thisAssignments.js create mode 100644 javascript/ql/test/query-tests/Security/CWE-079/addEventListener.js diff --git a/javascript/ql/src/semmle/javascript/Expr.qll b/javascript/ql/src/semmle/javascript/Expr.qll index 3640ee50a62..6cfa72ea878 100644 --- a/javascript/ql/src/semmle/javascript/Expr.qll +++ b/javascript/ql/src/semmle/javascript/Expr.qll @@ -303,6 +303,16 @@ class ThisExpr extends @thisexpr, Expr { Function getBinder() { result = getEnclosingFunction().getThisBinder() } + + /** + * Gets the function or top-level whose `this` binding this expression refers to, + * which is the nearest enclosing non-arrow function or top-level. + */ + StmtContainer getBindingContainer() { + result = getContainer().(Function).getThisBindingContainer() + or + result = getContainer().(TopLevel) + } } /** An array literal. */ diff --git a/javascript/ql/src/semmle/javascript/Functions.qll b/javascript/ql/src/semmle/javascript/Functions.qll index 1bd8aace891..049b66128a4 100644 --- a/javascript/ql/src/semmle/javascript/Functions.qll +++ b/javascript/ql/src/semmle/javascript/Functions.qll @@ -206,6 +206,17 @@ class Function extends @function, Parameterized, TypeParameterized, StmtContaine result = this } + /** + * Gets the function or top-level whose `this` binding a `this` expression in this function refers to, + * which is the nearest enclosing non-arrow function or top-level. + */ + StmtContainer getThisBindingContainer() { + result = getThisBinder() + or + not exists(getThisBinder()) and + result = getTopLevel() + } + /** * Holds if this function has a mapped `arguments` variable whose indices are aliased * with the function's parameters. diff --git a/javascript/ql/src/semmle/javascript/dataflow/DataFlow.qll b/javascript/ql/src/semmle/javascript/dataflow/DataFlow.qll index 7686f7e323e..ecf73bccde1 100644 --- a/javascript/ql/src/semmle/javascript/dataflow/DataFlow.qll +++ b/javascript/ql/src/semmle/javascript/dataflow/DataFlow.qll @@ -32,6 +32,7 @@ module DataFlow { or TReflectiveCallNode(MethodCallExpr ce, string kind) { ce.getMethodName() = kind and (kind = "call" or kind = "apply") } + or TThisNode(StmtContainer f) { f.(Function).getThisBinder() = f or f instanceof TopLevel } /** * A node in the data flow graph. @@ -867,6 +868,13 @@ module DataFlow { nd = TDestructuringPatternNode(p) } + /** + * INTERNAL: Use `thisNode(StmtContainer container)` instead. + */ + predicate thisNode(DataFlow::Node node, StmtContainer container) { + node = TThisNode(container) + } + /** * A classification of flows that are not modeled, or only modeled incompletely, by * `DataFlowNode`: @@ -970,6 +978,11 @@ module DataFlow { pred = valueNode(defSourceNode(def)) and succ = TDestructuringPatternNode(def.getTarget()) ) + or + // flow from 'this' parameter into 'this' expressions + exists (ThisExpr thiz | + pred = TThisNode(thiz.getBindingContainer()) and + succ = valueNode(thiz)) } /** diff --git a/javascript/ql/src/semmle/javascript/dataflow/Nodes.qll b/javascript/ql/src/semmle/javascript/dataflow/Nodes.qll index 24eec50029a..a20d8105bb1 100644 --- a/javascript/ql/src/semmle/javascript/dataflow/Nodes.qll +++ b/javascript/ql/src/semmle/javascript/dataflow/Nodes.qll @@ -198,16 +198,36 @@ class NewNode extends InvokeNode { override DataFlow::Impl::NewNodeDef impl; } -/** A data flow node corresponding to a `this` expression. */ -class ThisNode extends DataFlow::ValueNode, DataFlow::DefaultSourceNode { - override ThisExpr astNode; +/** A data flow node corresponding to the `this` parameter in a function or `this` at the top-level. */ +class ThisNode extends DataFlow::Node, DataFlow::DefaultSourceNode { + ThisNode() { + DataFlow::thisNode(this, _) + } /** * Gets the function whose `this` binding this expression refers to, * which is the nearest enclosing non-arrow function. */ FunctionNode getBinder() { - result = DataFlow::valueNode(astNode.getBinder()) + exists (Function binder | + DataFlow::thisNode(this, binder) and + result = DataFlow::valueNode(binder)) + } + + /** + * Gets the function or top-level whose `this` binding this expression refers to, + * which is the nearest enclosing non-arrow function or top-level. + */ + StmtContainer getBindingContainer() { + DataFlow::thisNode(this, result) + } + + override string toString() { result = "this" } + + override predicate hasLocationInfo(string filepath, int startline, int startcolumn, + int endline, int endcolumn) { + // Use the function entry as the location + getBindingContainer().getEntry().getLocation().hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn) } } diff --git a/javascript/ql/src/semmle/javascript/dataflow/Sources.qll b/javascript/ql/src/semmle/javascript/dataflow/Sources.qll index b4fa213f955..44f773f3b2d 100644 --- a/javascript/ql/src/semmle/javascript/dataflow/Sources.qll +++ b/javascript/ql/src/semmle/javascript/dataflow/Sources.qll @@ -185,7 +185,6 @@ class DefaultSourceNode extends SourceNode { astNode instanceof ObjectExpr or astNode instanceof ArrayExpr or astNode instanceof JSXNode or - astNode instanceof ThisExpr or astNode instanceof GlobalVarAccess or astNode instanceof ExternalModuleReference ) @@ -198,5 +197,7 @@ class DefaultSourceNode extends SourceNode { DataFlow::parameterNode(this, _) or this instanceof DataFlow::Impl::InvokeNodeDef + or + DataFlow::thisNode(this, _) } } diff --git a/javascript/ql/src/semmle/javascript/frameworks/React.qll b/javascript/ql/src/semmle/javascript/frameworks/React.qll index 316955ca77d..f4bd0671163 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/React.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/React.qll @@ -52,10 +52,10 @@ abstract class ReactComponent extends ASTNode { } /** - * Gets a `this` access in an instance method of this component. + * Gets the `this` node in an instance method of this component. */ DataFlow::SourceNode getAThisAccess() { - result.asExpr().(ThisExpr).getBinder() = getInstanceMethod(_) + result.(DataFlow::ThisNode).getBinder().getFunction() = getInstanceMethod(_) } /** diff --git a/javascript/ql/test/library-tests/TaintTracking/BasicTaintTracking.expected b/javascript/ql/test/library-tests/TaintTracking/BasicTaintTracking.expected index 0e052f99d27..2e0eb1ec1ef 100644 --- a/javascript/ql/test/library-tests/TaintTracking/BasicTaintTracking.expected +++ b/javascript/ql/test/library-tests/TaintTracking/BasicTaintTracking.expected @@ -3,6 +3,8 @@ | partialCalls.js:4:17:4:24 | source() | partialCalls.js:30:14:30:20 | x.value | | partialCalls.js:4:17:4:24 | source() | partialCalls.js:41:10:41:18 | id(taint) | | partialCalls.js:4:17:4:24 | source() | partialCalls.js:51:14:51:14 | x | +| thisAssignments.js:4:17:4:24 | source() | thisAssignments.js:5:10:5:18 | obj.field | +| thisAssignments.js:7:19:7:26 | source() | thisAssignments.js:8:10:8:20 | this.field2 | | tst.js:2:13:2:20 | source() | tst.js:4:10:4:10 | x | | tst.js:2:13:2:20 | source() | tst.js:5:10:5:22 | "/" + x + "!" | | tst.js:2:13:2:20 | source() | tst.js:14:10:14:17 | x.sort() | diff --git a/javascript/ql/test/library-tests/TaintTracking/thisAssignments.js b/javascript/ql/test/library-tests/TaintTracking/thisAssignments.js new file mode 100644 index 00000000000..cfe5b0c6e0a --- /dev/null +++ b/javascript/ql/test/library-tests/TaintTracking/thisAssignments.js @@ -0,0 +1,10 @@ +class C { + foo() { + let obj = {}; + obj.field = source(); + sink(obj.field); // NOT OK - tainted + + this.field2 = source(); + sink(this.field2); // NOT OK - tainted + } +} diff --git a/javascript/ql/test/query-tests/Security/CWE-079/Xss.expected b/javascript/ql/test/query-tests/Security/CWE-079/Xss.expected index b3ba31e89dc..00088f35f8f 100644 --- a/javascript/ql/test/query-tests/Security/CWE-079/Xss.expected +++ b/javascript/ql/test/query-tests/Security/CWE-079/Xss.expected @@ -1,3 +1,4 @@ +| addEventListener.js:2:20:2:29 | event.data | Cross-site scripting vulnerability due to $@. | addEventListener.js:1:43:1:47 | event | user-provided value | | jquery.js:4:5:4:11 | tainted | Cross-site scripting vulnerability due to $@. | jquery.js:2:17:2:33 | document.location | user-provided value | | jquery.js:7:5:7:34 | "
" | Cross-site scripting vulnerability due to $@. | jquery.js:2:17:2:33 | document.location | user-provided value | | jquery.js:8:18:8:34 | "XSS: " + tainted | Cross-site scripting vulnerability due to $@. | jquery.js:2:17:2:33 | document.location | user-provided value | diff --git a/javascript/ql/test/query-tests/Security/CWE-079/addEventListener.js b/javascript/ql/test/query-tests/Security/CWE-079/addEventListener.js new file mode 100644 index 00000000000..2f4b92adbee --- /dev/null +++ b/javascript/ql/test/query-tests/Security/CWE-079/addEventListener.js @@ -0,0 +1,3 @@ +this.addEventListener('message', function(event) { + document.write(event.data); // NOT OK +}) From e551ff38189af22a0808f6e9990f6f311cda350b Mon Sep 17 00:00:00 2001 From: Asger F Date: Mon, 8 Oct 2018 11:48:21 +0100 Subject: [PATCH 28/58] JS: add change note --- change-notes/1.19/analysis-javascript.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/change-notes/1.19/analysis-javascript.md b/change-notes/1.19/analysis-javascript.md index d3b2958d874..0dced020616 100644 --- a/change-notes/1.19/analysis-javascript.md +++ b/change-notes/1.19/analysis-javascript.md @@ -35,4 +35,6 @@ ## Changes to QL libraries -* The flow configuration framework now supports distinguishing and tracking different kinds of taint, specified by an extensible class `FlowLabel` (which can also be referred to by its alias `TaintKind`). \ No newline at end of file +* The flow configuration framework now supports distinguishing and tracking different kinds of taint, specified by an extensible class `FlowLabel` (which can also be referred to by its alias `TaintKind`). + +* The `DataFlow::ThisNode` class now corresponds to the implicit receiver parameter of a function, as opposed to an indivdual `this` expression. This means `getALocalSource` now maps all `this` expressions within a given function to the same source. The data-flow node associated with a `ThisExpr` can no longer be cast to `DataFlow::SourceNode` or `DataFlow::ThisNode` - it is recomended to use `getALocalSource` before casting or instead of casting. From fd58039753ac5a2d5375ad55347f1420931643f4 Mon Sep 17 00:00:00 2001 From: Asger F Date: Tue, 9 Oct 2018 08:16:54 +0100 Subject: [PATCH 29/58] JS: update additional QL test output --- .../library-tests/DataFlow/flowStep.expected | 1 + .../library-tests/DataFlow/sources.expected | 17 +++++++- .../Nodes/globalObjectRef.expected | 4 +- .../PropWrite/getAPropertyRead.expected | 4 +- .../PropWrite/getAPropertyRead2.expected | 4 +- .../PropWrite/getAPropertyReference.expected | 4 +- .../PropWrite/getAPropertyReference2.expected | 4 +- .../ReactJS/ReactComponent_ref.expected | 41 +++++++++++++++++++ 8 files changed, 68 insertions(+), 11 deletions(-) diff --git a/javascript/ql/test/library-tests/DataFlow/flowStep.expected b/javascript/ql/test/library-tests/DataFlow/flowStep.expected index 55ade0c3093..2320ec3a36b 100644 --- a/javascript/ql/test/library-tests/DataFlow/flowStep.expected +++ b/javascript/ql/test/library-tests/DataFlow/flowStep.expected @@ -76,6 +76,7 @@ | tst.js:37:5:42:1 | o | tst.js:83:23:83:23 | o | | tst.js:37:5:42:1 | o | tst.js:85:23:85:23 | o | | tst.js:37:9:42:1 | {\\n x: ... ;\\n }\\n} | tst.js:37:5:42:1 | o | +| tst.js:39:4:39:3 | this | tst.js:40:5:40:8 | this | | tst.js:46:10:46:11 | "" | tst.js:46:1:46:11 | global = "" | | tst.js:49:1:54:1 | A | tst.js:55:1:55:1 | A | | tst.js:49:1:54:1 | class A ... `\\n }\\n} | tst.js:49:1:54:1 | A | diff --git a/javascript/ql/test/library-tests/DataFlow/sources.expected b/javascript/ql/test/library-tests/DataFlow/sources.expected index 5c22f3db1c7..8aa26a488f0 100644 --- a/javascript/ql/test/library-tests/DataFlow/sources.expected +++ b/javascript/ql/test/library-tests/DataFlow/sources.expected @@ -1,14 +1,20 @@ +| eval.js:1:1:1:0 | this | +| eval.js:1:1:1:0 | this | | eval.js:1:1:5:1 | functio ... eval`\\n} | | eval.js:3:3:3:6 | eval | | eval.js:3:3:3:16 | eval("x = 23") | +| sources.js:1:1:1:0 | this | | sources.js:1:1:1:12 | new (x => x) | | sources.js:1:6:1:6 | x | | sources.js:1:6:1:11 | x => x | | sources.js:3:1:5:6 | (functi ... \\n})(23) | +| sources.js:3:2:3:1 | this | | sources.js:3:2:5:1 | functio ... x+19;\\n} | | sources.js:3:11:3:11 | x | +| tst.js:1:1:1:0 | this | | tst.js:1:10:1:11 | fs | | tst.js:16:1:20:9 | (functi ... ("arg") | +| tst.js:16:2:16:1 | this | | tst.js:16:2:20:1 | functio ... n "";\\n} | | tst.js:16:13:16:13 | a | | tst.js:17:7:17:10 | Math | @@ -17,11 +23,12 @@ | tst.js:22:7:22:18 | readFileSync | | tst.js:28:1:30:3 | (() =>\\n ... les\\n)() | | tst.js:28:2:29:3 | () =>\\n x | +| tst.js:32:1:32:0 | this | | tst.js:32:1:34:1 | functio ... ables\\n} | | tst.js:35:1:35:7 | g(true) | | tst.js:37:9:42:1 | {\\n x: ... ;\\n }\\n} | +| tst.js:39:4:39:3 | this | | tst.js:39:4:41:3 | () {\\n this;\\n } | -| tst.js:40:5:40:8 | this | | tst.js:43:1:43:3 | o.x | | tst.js:44:1:44:3 | o.m | | tst.js:44:1:44:5 | o.m() | @@ -29,18 +36,22 @@ | tst.js:47:1:47:6 | global | | tst.js:49:1:54:1 | class A ... `\\n }\\n} | | tst.js:49:17:49:17 | B | +| tst.js:50:14:50:13 | this | | tst.js:50:14:53:3 | () {\\n ... et`\\n } | | tst.js:51:5:51:13 | super(42) | | tst.js:58:1:58:3 | tag | | tst.js:61:3:61:5 | o.m | +| tst.js:64:1:64:0 | this | | tst.js:64:1:67:1 | functio ... lysed\\n} | | tst.js:68:12:68:14 | h() | | tst.js:69:1:69:9 | iter.next | | tst.js:69:1:69:13 | iter.next(23) | +| tst.js:71:1:71:0 | this | | tst.js:71:1:73:1 | async f ... lysed\\n} | | tst.js:72:9:72:9 | p | | tst.js:72:9:72:11 | p() | | tst.js:87:1:96:2 | (functi ... r: 0\\n}) | +| tst.js:87:2:87:1 | this | | tst.js:87:2:92:1 | functio ... + z;\\n} | | tst.js:87:11:87:24 | { p: x, ...o } | | tst.js:87:13:87:16 | p: x | @@ -49,6 +60,7 @@ | tst.js:90:6:90:9 | r: z | | tst.js:92:4:96:1 | {\\n p: ... r: 0\\n} | | tst.js:98:1:103:17 | (functi ... 3, 0 ]) | +| tst.js:98:2:98:1 | this | | tst.js:98:2:103:1 | functio ... + z;\\n} | | tst.js:98:11:98:24 | [ x, ...rest ] | | tst.js:98:13:98:13 | x | @@ -56,7 +68,9 @@ | tst.js:99:9:99:9 | y | | tst.js:101:7:101:7 | z | | tst.js:103:4:103:16 | [ 19, 23, 0 ] | +| tst.ts:1:1:1:0 | this | | tst.ts:3:3:3:8 | setX() | +| tst.ts:7:1:7:0 | this | | tst.ts:7:1:9:1 | functio ... = 23;\\n} | | tst.ts:8:3:8:5 | A.x | | tst.ts:11:11:11:13 | A.x | @@ -65,3 +79,4 @@ | tst.ts:13:39:13:38 | (...arg ... rgs); } | | tst.ts:13:39:13:38 | args | | tst.ts:13:39:13:38 | super(...args) | +| tst.ts:13:39:13:38 | this | diff --git a/javascript/ql/test/library-tests/Nodes/globalObjectRef.expected b/javascript/ql/test/library-tests/Nodes/globalObjectRef.expected index 81e2ff22808..b8696f4136d 100644 --- a/javascript/ql/test/library-tests/Nodes/globalObjectRef.expected +++ b/javascript/ql/test/library-tests/Nodes/globalObjectRef.expected @@ -1,10 +1,10 @@ +| tst2.js:1:1:1:0 | this | | tst2.js:1:9:1:25 | require("global") | | tst2.js:3:1:3:24 | require ... indow") | | tst2.js:7:1:7:6 | global | | tst2.js:8:1:8:6 | global | -| tst2.js:9:1:9:4 | this | +| tst.js:1:1:1:0 | this | | tst.js:1:1:1:6 | window | -| tst.js:2:1:2:4 | this | | tst.js:3:1:3:6 | window | | tst.js:4:1:4:6 | window | | tst.js:4:1:4:13 | window.window | diff --git a/javascript/ql/test/library-tests/PropWrite/getAPropertyRead.expected b/javascript/ql/test/library-tests/PropWrite/getAPropertyRead.expected index 1ee35008e8a..bfce597c047 100644 --- a/javascript/ql/test/library-tests/PropWrite/getAPropertyRead.expected +++ b/javascript/ql/test/library-tests/PropWrite/getAPropertyRead.expected @@ -1,8 +1,8 @@ +| tst.js:1:1:1:0 | this | tst.js:23:15:23:29 | this.someMethod | +| tst.js:1:1:1:0 | this | tst.js:24:36:24:45 | this.state | | tst.js:14:5:14:11 | console | tst.js:14:5:14:15 | console.log | | tst.js:17:5:17:11 | console | tst.js:17:5:17:15 | console.log | -| tst.js:23:15:23:18 | this | tst.js:23:15:23:29 | this.someMethod | | tst.js:23:15:23:29 | this.someMethod | tst.js:23:15:23:34 | this.someMethod.bind | -| tst.js:24:36:24:39 | this | tst.js:24:36:24:45 | this.state | | tst.js:24:36:24:45 | this.state | tst.js:24:36:24:50 | this.state.name | | tst.js:34:6:34:7 | vv | tst.js:34:6:34:10 | vv.pp | | tst.js:35:6:35:8 | vvv | tst.js:35:6:35:12 | vvv.ppp | diff --git a/javascript/ql/test/library-tests/PropWrite/getAPropertyRead2.expected b/javascript/ql/test/library-tests/PropWrite/getAPropertyRead2.expected index 28d026931f3..854d5931849 100644 --- a/javascript/ql/test/library-tests/PropWrite/getAPropertyRead2.expected +++ b/javascript/ql/test/library-tests/PropWrite/getAPropertyRead2.expected @@ -1,8 +1,8 @@ +| tst.js:1:1:1:0 | this | someMethod | tst.js:23:15:23:29 | this.someMethod | +| tst.js:1:1:1:0 | this | state | tst.js:24:36:24:45 | this.state | | tst.js:14:5:14:11 | console | log | tst.js:14:5:14:15 | console.log | | tst.js:17:5:17:11 | console | log | tst.js:17:5:17:15 | console.log | -| tst.js:23:15:23:18 | this | someMethod | tst.js:23:15:23:29 | this.someMethod | | tst.js:23:15:23:29 | this.someMethod | bind | tst.js:23:15:23:34 | this.someMethod.bind | -| tst.js:24:36:24:39 | this | state | tst.js:24:36:24:45 | this.state | | tst.js:24:36:24:45 | this.state | name | tst.js:24:36:24:50 | this.state.name | | tst.js:34:6:34:7 | vv | pp | tst.js:34:6:34:10 | vv.pp | | tst.js:35:6:35:8 | vvv | ppp | tst.js:35:6:35:12 | vvv.ppp | diff --git a/javascript/ql/test/library-tests/PropWrite/getAPropertyReference.expected b/javascript/ql/test/library-tests/PropWrite/getAPropertyReference.expected index 2faf239e154..e31be10e629 100644 --- a/javascript/ql/test/library-tests/PropWrite/getAPropertyReference.expected +++ b/javascript/ql/test/library-tests/PropWrite/getAPropertyReference.expected @@ -1,3 +1,5 @@ +| tst.js:1:1:1:0 | this | tst.js:23:15:23:29 | this.someMethod | +| tst.js:1:1:1:0 | this | tst.js:24:36:24:45 | this.state | | tst.js:2:11:10:1 | {\\n x ... }\\n} | tst.js:3:5:3:8 | x: 4 | | tst.js:2:11:10:1 | {\\n x ... }\\n} | tst.js:4:5:6:5 | func: f ... ;\\n } | | tst.js:2:11:10:1 | {\\n x ... }\\n} | tst.js:7:5:9:5 | f() {\\n ... ;\\n } | @@ -5,10 +7,8 @@ | tst.js:14:5:14:11 | console | tst.js:14:5:14:15 | console.log | | tst.js:17:5:17:11 | console | tst.js:17:5:17:15 | console.log | | tst.js:21:1:21:1 | C | tst.js:21:1:21:6 | C.prop | -| tst.js:23:15:23:18 | this | tst.js:23:15:23:29 | this.someMethod | | tst.js:23:15:23:29 | this.someMethod | tst.js:23:15:23:34 | this.someMethod.bind | | tst.js:24:8:24:57 |
| tst.js:24:13:24:27 | onClick={click} | -| tst.js:24:36:24:39 | this | tst.js:24:36:24:45 | this.state | | tst.js:24:36:24:45 | this.state | tst.js:24:36:24:50 | this.state.name | | tst.js:26:2:29:1 | {\\n get ... v) {}\\n} | tst.js:27:3:27:26 | get x() ... null; } | | tst.js:26:2:29:1 | {\\n get ... v) {}\\n} | tst.js:28:3:28:13 | set y(v) {} | diff --git a/javascript/ql/test/library-tests/PropWrite/getAPropertyReference2.expected b/javascript/ql/test/library-tests/PropWrite/getAPropertyReference2.expected index 58747eac5eb..1f18e7dd51e 100644 --- a/javascript/ql/test/library-tests/PropWrite/getAPropertyReference2.expected +++ b/javascript/ql/test/library-tests/PropWrite/getAPropertyReference2.expected @@ -1,3 +1,5 @@ +| tst.js:1:1:1:0 | this | someMethod | tst.js:23:15:23:29 | this.someMethod | +| tst.js:1:1:1:0 | this | state | tst.js:24:36:24:45 | this.state | | tst.js:2:11:10:1 | {\\n x ... }\\n} | f | tst.js:7:5:9:5 | f() {\\n ... ;\\n } | | tst.js:2:11:10:1 | {\\n x ... }\\n} | func | tst.js:4:5:6:5 | func: f ... ;\\n } | | tst.js:2:11:10:1 | {\\n x ... }\\n} | x | tst.js:3:5:3:8 | x: 4 | @@ -5,10 +7,8 @@ | tst.js:14:5:14:11 | console | log | tst.js:14:5:14:15 | console.log | | tst.js:17:5:17:11 | console | log | tst.js:17:5:17:15 | console.log | | tst.js:21:1:21:1 | C | prop | tst.js:21:1:21:6 | C.prop | -| tst.js:23:15:23:18 | this | someMethod | tst.js:23:15:23:29 | this.someMethod | | tst.js:23:15:23:29 | this.someMethod | bind | tst.js:23:15:23:34 | this.someMethod.bind | | tst.js:24:8:24:57 |
| onClick | tst.js:24:13:24:27 | onClick={click} | -| tst.js:24:36:24:39 | this | state | tst.js:24:36:24:45 | this.state | | tst.js:24:36:24:45 | this.state | name | tst.js:24:36:24:50 | this.state.name | | tst.js:26:2:29:1 | {\\n get ... v) {}\\n} | x | tst.js:27:3:27:26 | get x() ... null; } | | tst.js:26:2:29:1 | {\\n get ... v) {}\\n} | y | tst.js:28:3:28:13 | set y(v) {} | diff --git a/javascript/ql/test/library-tests/frameworks/ReactJS/ReactComponent_ref.expected b/javascript/ql/test/library-tests/frameworks/ReactJS/ReactComponent_ref.expected index fb34433b683..8daacce1a23 100644 --- a/javascript/ql/test/library-tests/frameworks/ReactJS/ReactComponent_ref.expected +++ b/javascript/ql/test/library-tests/frameworks/ReactJS/ReactComponent_ref.expected @@ -1,18 +1,42 @@ | es5.js:1:31:11:1 | {\\n dis ... ;\\n }\\n} | es5.js:1:31:11:1 | {\\n dis ... ;\\n }\\n} | +| es5.js:1:31:11:1 | {\\n dis ... ;\\n }\\n} | es5.js:3:11:3:10 | this | | es5.js:1:31:11:1 | {\\n dis ... ;\\n }\\n} | es5.js:4:24:4:27 | this | +| es5.js:1:31:11:1 | {\\n dis ... ;\\n }\\n} | es5.js:6:20:6:19 | this | | es5.js:18:33:22:1 | {\\n ren ... ;\\n }\\n} | es5.js:18:33:22:1 | {\\n ren ... ;\\n }\\n} | +| es5.js:18:33:22:1 | {\\n ren ... ;\\n }\\n} | es5.js:19:11:19:10 | this | | es5.js:18:33:22:1 | {\\n ren ... ;\\n }\\n} | es5.js:20:24:20:27 | this | +| es6.js:1:1:8:1 | class H ... ;\\n }\\n} | es6.js:1:37:1:36 | this | +| es6.js:1:1:8:1 | class H ... ;\\n }\\n} | es6.js:2:9:2:8 | this | | es6.js:1:1:8:1 | class H ... ;\\n }\\n} | es6.js:3:24:3:27 | this | +| es6.js:1:1:8:1 | class H ... ;\\n }\\n} | es6.js:5:14:5:13 | this | +| es6.js:14:1:20:1 | class H ... }\\n} | es6.js:15:16:15:15 | this | | es6.js:14:1:20:1 | class H ... }\\n} | es6.js:16:9:16:12 | this | | es6.js:14:1:20:1 | class H ... }\\n} | es6.js:17:9:17:12 | this | | es6.js:14:1:20:1 | class H ... }\\n} | es6.js:18:9:18:12 | this | +| namedImport.js:3:1:3:28 | class C ... nent {} | namedImport.js:3:27:3:26 | this | +| namedImport.js:5:1:5:20 | class D extends C {} | namedImport.js:5:19:5:18 | this | +| plainfn.js:1:1:3:1 | functio ... div>;\\n} | plainfn.js:1:1:1:0 | this | +| plainfn.js:5:1:7:1 | functio ... iv");\\n} | plainfn.js:5:1:5:0 | this | +| plainfn.js:9:1:12:1 | functio ... rn x;\\n} | plainfn.js:9:1:9:0 | this | +| plainfn.js:20:1:24:1 | functio ... n 42;\\n} | plainfn.js:20:1:20:0 | this | +| preact.js:1:1:7:1 | class H ... }\\n} | preact.js:1:38:1:37 | this | +| preact.js:1:1:7:1 | class H ... }\\n} | preact.js:2:11:2:10 | this | +| preact.js:9:1:11:1 | class H ... nt {\\n\\n} | preact.js:9:38:9:37 | this | +| probably-a-component.js:1:1:6:1 | class H ... }\\n} | probably-a-component.js:1:31:1:30 | this | +| probably-a-component.js:1:1:6:1 | class H ... }\\n} | probably-a-component.js:2:11:2:10 | this | | probably-a-component.js:1:1:6:1 | class H ... }\\n} | probably-a-component.js:3:9:3:12 | this | +| props.js:2:5:3:5 | class C ... {\\n } | props.js:2:37:2:36 | this | | props.js:2:5:3:5 | class C ... {\\n } | props.js:9:5:9:55 | new C({ ... ctor"}) | | props.js:13:31:17:5 | {\\n ... }\\n } | props.js:13:31:17:5 | {\\n ... }\\n } | +| props.js:13:31:17:5 | {\\n ... }\\n } | props.js:14:24:14:23 | this | +| props.js:26:5:28:5 | functio ... ;\\n } | props.js:26:5:26:4 | this | | props.js:26:5:28:5 | functio ... ;\\n } | props.js:34:5:34:55 | new C({ ... ctor"}) | +| statePropertyReads.js:1:1:13:1 | class R ... }\\n} | statePropertyReads.js:2:16:2:15 | this | | statePropertyReads.js:1:1:13:1 | class R ... }\\n} | statePropertyReads.js:3:9:3:12 | this | | statePropertyReads.js:1:1:13:1 | class R ... }\\n} | statePropertyReads.js:5:9:5:12 | this | | statePropertyReads.js:1:1:13:1 | class R ... }\\n} | statePropertyReads.js:7:9:7:12 | this | +| statePropertyReads.js:1:1:13:1 | class R ... }\\n} | statePropertyReads.js:10:23:10:22 | this | +| statePropertyWrites.js:1:1:34:1 | class W ... };\\n} | statePropertyWrites.js:2:16:2:15 | this | | statePropertyWrites.js:1:1:34:1 | class W ... };\\n} | statePropertyWrites.js:3:13:3:22 | cmp | | statePropertyWrites.js:1:1:34:1 | class W ... };\\n} | statePropertyWrites.js:3:19:3:22 | this | | statePropertyWrites.js:1:1:34:1 | class W ... };\\n} | statePropertyWrites.js:4:9:4:11 | cmp | @@ -21,28 +45,45 @@ | statePropertyWrites.js:1:1:34:1 | class W ... };\\n} | statePropertyWrites.js:14:9:14:11 | cmp | | statePropertyWrites.js:1:1:34:1 | class W ... };\\n} | statePropertyWrites.js:18:9:18:11 | cmp | | statePropertyWrites.js:1:1:34:1 | class W ... };\\n} | statePropertyWrites.js:22:9:22:11 | cmp | +| statePropertyWrites.js:1:1:34:1 | class W ... };\\n} | statePropertyWrites.js:25:20:25:19 | this | | statePropertyWrites.js:36:19:45:1 | {\\n ren ... ;\\n }\\n} | statePropertyWrites.js:36:19:45:1 | {\\n ren ... ;\\n }\\n} | +| statePropertyWrites.js:36:19:45:1 | {\\n ren ... ;\\n }\\n} | statePropertyWrites.js:37:11:37:10 | this | | statePropertyWrites.js:36:19:45:1 | {\\n ren ... ;\\n }\\n} | statePropertyWrites.js:38:24:38:27 | this | +| statePropertyWrites.js:36:19:45:1 | {\\n ren ... ;\\n }\\n} | statePropertyWrites.js:40:20:40:19 | this | +| thisAccesses.js:1:1:16:1 | class C ... }\\n} | thisAccesses.js:2:17:2:16 | this | | thisAccesses.js:1:1:16:1 | class C ... }\\n} | thisAccesses.js:3:9:3:12 | this | | thisAccesses.js:1:1:16:1 | class C ... }\\n} | thisAccesses.js:5:13:5:22 | dis | | thisAccesses.js:1:1:16:1 | class C ... }\\n} | thisAccesses.js:5:19:5:22 | this | | thisAccesses.js:1:1:16:1 | class C ... }\\n} | thisAccesses.js:6:9:6:11 | dis | +| thisAccesses.js:1:1:16:1 | class C ... }\\n} | thisAccesses.js:8:10:8:9 | this | | thisAccesses.js:1:1:16:1 | class C ... }\\n} | thisAccesses.js:9:13:9:16 | this | | thisAccesses.js:1:1:16:1 | class C ... }\\n} | thisAccesses.js:10:17:10:20 | this | +| thisAccesses.js:1:1:16:1 | class C ... }\\n} | thisAccesses.js:13:23:13:22 | this | | thisAccesses.js:1:1:16:1 | class C ... }\\n} | thisAccesses.js:14:9:14:12 | this | | thisAccesses.js:18:19:29:1 | {\\n r ... }\\n} | thisAccesses.js:18:19:29:1 | {\\n r ... }\\n} | +| thisAccesses.js:18:19:29:1 | {\\n r ... }\\n} | thisAccesses.js:19:13:19:12 | this | +| thisAccesses.js:18:19:29:1 | {\\n r ... }\\n} | thisAccesses.js:20:10:20:9 | this | | thisAccesses.js:18:19:29:1 | {\\n r ... }\\n} | thisAccesses.js:21:13:21:16 | this | | thisAccesses.js:18:19:29:1 | {\\n r ... }\\n} | thisAccesses.js:22:17:22:20 | this | +| thisAccesses.js:18:19:29:1 | {\\n r ... }\\n} | thisAccesses.js:26:25:26:24 | this | | thisAccesses.js:18:19:29:1 | {\\n r ... }\\n} | thisAccesses.js:27:9:27:12 | this | +| thisAccesses.js:31:2:36:1 | functio ... iv/>;\\n} | thisAccesses.js:31:2:31:1 | this | +| thisAccesses.js:31:2:36:1 | functio ... iv/>;\\n} | thisAccesses.js:32:6:32:5 | this | | thisAccesses.js:31:2:36:1 | functio ... iv/>;\\n} | thisAccesses.js:33:9:33:12 | this | | thisAccesses.js:31:2:36:1 | functio ... iv/>;\\n} | thisAccesses.js:34:13:34:16 | this | | thisAccesses.js:38:19:45:1 | {\\n r ... },\\n} | thisAccesses.js:38:19:45:1 | {\\n r ... },\\n} | +| thisAccesses.js:38:19:45:1 | {\\n r ... },\\n} | thisAccesses.js:39:13:39:12 | this | +| thisAccesses.js:38:19:45:1 | {\\n r ... },\\n} | thisAccesses.js:40:38:40:37 | this | | thisAccesses.js:38:19:45:1 | {\\n r ... },\\n} | thisAccesses.js:41:13:41:16 | this | | thisAccesses.js:38:19:45:1 | {\\n r ... },\\n} | thisAccesses.js:42:12:42:15 | this | +| thisAccesses.js:47:1:52:1 | class C ... }\\n} | thisAccesses.js:48:17:48:16 | this | | thisAccesses.js:47:1:52:1 | class C ... }\\n} | thisAccesses.js:49:9:49:12 | this | | thisAccesses.js:47:1:52:1 | class C ... }\\n} | thisAccesses.js:50:9:50:12 | this | | thisAccesses_importedMappers.js:4:19:15:1 | {\\n r ... },\\n} | thisAccesses_importedMappers.js:4:19:15:1 | {\\n r ... },\\n} | +| thisAccesses_importedMappers.js:4:19:15:1 | {\\n r ... },\\n} | thisAccesses_importedMappers.js:5:13:5:12 | this | +| thisAccesses_importedMappers.js:4:19:15:1 | {\\n r ... },\\n} | thisAccesses_importedMappers.js:6:38:6:37 | this | | thisAccesses_importedMappers.js:4:19:15:1 | {\\n r ... },\\n} | thisAccesses_importedMappers.js:7:13:7:16 | this | | thisAccesses_importedMappers.js:4:19:15:1 | {\\n r ... },\\n} | thisAccesses_importedMappers.js:8:12:8:15 | this | +| thisAccesses_importedMappers.js:4:19:15:1 | {\\n r ... },\\n} | thisAccesses_importedMappers.js:9:25:9:24 | this | | thisAccesses_importedMappers.js:4:19:15:1 | {\\n r ... },\\n} | thisAccesses_importedMappers.js:10:13:10:16 | this | | thisAccesses_importedMappers.js:4:19:15:1 | {\\n r ... },\\n} | thisAccesses_importedMappers.js:11:12:11:15 | this | From 9fb73f41c912af2923c26da72b2ea944ba249572 Mon Sep 17 00:00:00 2001 From: Asger F Date: Tue, 9 Oct 2018 08:48:56 +0100 Subject: [PATCH 30/58] JS: rename ReactComponent::getAThisAccess -> getAThisNode --- change-notes/1.19/analysis-javascript.md | 2 ++ .../ql/src/semmle/javascript/frameworks/React.qll | 12 +++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/change-notes/1.19/analysis-javascript.md b/change-notes/1.19/analysis-javascript.md index 0dced020616..bf36867b2eb 100644 --- a/change-notes/1.19/analysis-javascript.md +++ b/change-notes/1.19/analysis-javascript.md @@ -38,3 +38,5 @@ * The flow configuration framework now supports distinguishing and tracking different kinds of taint, specified by an extensible class `FlowLabel` (which can also be referred to by its alias `TaintKind`). * The `DataFlow::ThisNode` class now corresponds to the implicit receiver parameter of a function, as opposed to an indivdual `this` expression. This means `getALocalSource` now maps all `this` expressions within a given function to the same source. The data-flow node associated with a `ThisExpr` can no longer be cast to `DataFlow::SourceNode` or `DataFlow::ThisNode` - it is recomended to use `getALocalSource` before casting or instead of casting. + +* `ReactComponent::getAThisAccess` has been renamed to `getAThisNode`. The old name is still usable but is deprecated. It no longer gets individual `this` expressions, but the `ThisNode` mentioned above. diff --git a/javascript/ql/src/semmle/javascript/frameworks/React.qll b/javascript/ql/src/semmle/javascript/frameworks/React.qll index f4bd0671163..1ad07b9284b 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/React.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/React.qll @@ -54,10 +54,20 @@ abstract class ReactComponent extends ASTNode { /** * Gets the `this` node in an instance method of this component. */ - DataFlow::SourceNode getAThisAccess() { + DataFlow::SourceNode getAThisNode() { result.(DataFlow::ThisNode).getBinder().getFunction() = getInstanceMethod(_) } + /** + * Gets the `this` node in an instance method of this component. + * + * DEPRECATED: Use `getAThisNode` instead. + */ + deprecated + DataFlow::SourceNode getAThisAccess() { + result = getAThisNode() + } + /** * Gets an access to the `props` object of this component. * From 2d8f424ce89ae16ce300d4ed587d27a412f1ba0d Mon Sep 17 00:00:00 2001 From: Max Schaefer Date: Tue, 9 Oct 2018 13:01:00 +0100 Subject: [PATCH 31/58] C#: Convert `tests/query-tests/Stubs/Test.cs` to Unix line endings. --- csharp/ql/test/query-tests/Stubs/Test.cs | 250 +++++++++++------------ 1 file changed, 125 insertions(+), 125 deletions(-) diff --git a/csharp/ql/test/query-tests/Stubs/Test.cs b/csharp/ql/test/query-tests/Stubs/Test.cs index f0f0e6f82dd..a81faedcb1b 100644 --- a/csharp/ql/test/query-tests/Stubs/Test.cs +++ b/csharp/ql/test/query-tests/Stubs/Test.cs @@ -1,125 +1,125 @@ -// semmle-extractor-options: /r:System.Text.RegularExpressions.dll /r:System.Collections.Specialized.dll /r:System.Net.dll /r:System.Web.dll /r:System.Net.HttpListener.dll /r:System.Collections.Specialized.dll /r:System.Private.Uri.dll /r:System.Runtime.Extensions.dll /r:System.Linq.Parallel.dll /r:System.Collections.Concurrent.dll /r:System.Linq.Expressions.dll /r:System.Collections.dll /r:System.Linq.Queryable.dll /r:System.Linq.dll /r:System.Collections.NonGeneric.dll /r:System.ObjectModel.dll /r:System.ComponentModel.TypeConverter.dll /r:System.IO.Compression.dll /r:System.IO.Pipes.dll /r:System.Net.Primitives.dll /r:System.Net.Security.dll /r:System.Security.Cryptography.Primitives.dll /r:System.Text.RegularExpressions.dll ${testdir}/../../resources/stubs/System.Web.cs /r:System.Runtime.Serialization.Primitives.dll - -using System; -using System.IO; -using System.Text; -using System.Collections; -using System.Collections.Generic; -using System.Collections.Specialized; -using System.Linq; -using System.Runtime.Serialization; -using System.Threading.Tasks; -using System.Web; -using System.Web.UI.WebControls; -using System.Text.RegularExpressions; - -public class RegexHandler -{ - private static readonly string JAVA_CLASS_REGEX = "^(([a-z])+.)+[A-Z]([a-z])+$"; - - public void ProcessRequest() - { - string userInput = ""; - - // BAD: - // Artificial regexes - new Regex("^([a-z]+)+$").Match(userInput); - new Regex("^([a-z]*)*$").Replace(userInput, ""); - // Known exponential blowup regex for e-mail address validation - // Problematic part is: ([a-zA-Z0-9]+))* - new Regex("^([a-zA-Z0-9])(([\\-.]|[_]+)?([a-zA-Z0-9]+))*(@){1}[a-z0-9]+[.]{1}(([a-z]{2,3})|([a-z]{2,3}[.]{1}[a-z]{2,3}))$").Match(userInput); - // Known exponential blowup regex for Java class name validation - // Problematic part is: (([a-z])+.)+ - new Regex(JAVA_CLASS_REGEX).Match(userInput); - // Static use - Regex.Match(userInput, JAVA_CLASS_REGEX); - // GOOD: - new Regex("^(([a-b]+[c-z]+)+$").Match(userInput); - new Regex("^([a-z]+)+$", RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1)).Match(userInput); - Regex.Match(userInput, JAVA_CLASS_REGEX, RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1)); - // Known possible FP. - new Regex("^[a-z0-9]+([_.-][a-z0-9]+)*$").Match(userInput); - } -} - -// The only purpose of this class is to make sure the extractor extracts the -// relevant library methods -public class LibraryTypeDataFlow -{ - void M() - { - int i; - int.Parse(""); - int.TryParse("", out i); - - bool b; - bool.Parse(""); - bool.TryParse("", out b); - - Uri uri = null; - uri.ToString(); - - StringReader sr = new StringReader(""); - - string s = new string(new[] { 'a' }); - string.Join("", "", "", ""); - - StringBuilder sb = new StringBuilder(""); - - Lazy l = new Lazy(() => 42); - - IEnumerable ie = null; - ie.GetEnumerator(); - ie.AsParallel(); - ie.AsQueryable(); - IEnumerable ieint = null; - ieint.Select(x => x); - List list = null; - list.Find(x => x > 0); - Stack stack = null; - stack.Peek(); - ArrayList al = null; - ArrayList.FixedSize(al); - SortedList sl = null; - sl.GetByIndex(0); - - Convert.ToInt32("0"); - - DataContract dc = null; - s = dc.AString; - - KeyValuePair kvp = new KeyValuePair(0, ""); - - IEnumerator ienum = null; - object o = ienum.Current; - - IEnumerator ienumint = null; - i = ienumint.Current; - - var task = new Task(() => { }); - Task.WhenAll(null, null); - Task.WhenAny(null, null); - Task.Factory.ContinueWhenAll((Task[])null, (Func)null); - - var task2 = new Task(() => 42); - Task.Factory.ContinueWhenAny(new Task[] { task2 }, t => t.Result.ToString()); - - Encoding.Unicode.GetString(Encoding.Unicode.GetBytes("")); - - Path.Combine("", ""); - Path.GetDirectoryName(""); - Path.GetExtension(""); - Path.GetFileName(""); - Path.GetFileNameWithoutExtension(""); - Path.GetPathRoot(""); - HttpContextBase context = null; - string name = context.Request.QueryString["name"]; - } - - [DataContract] - public class DataContract - { - [DataMember] - public string AString { get; set; } - } -} +// semmle-extractor-options: /r:System.Text.RegularExpressions.dll /r:System.Collections.Specialized.dll /r:System.Net.dll /r:System.Web.dll /r:System.Net.HttpListener.dll /r:System.Collections.Specialized.dll /r:System.Private.Uri.dll /r:System.Runtime.Extensions.dll /r:System.Linq.Parallel.dll /r:System.Collections.Concurrent.dll /r:System.Linq.Expressions.dll /r:System.Collections.dll /r:System.Linq.Queryable.dll /r:System.Linq.dll /r:System.Collections.NonGeneric.dll /r:System.ObjectModel.dll /r:System.ComponentModel.TypeConverter.dll /r:System.IO.Compression.dll /r:System.IO.Pipes.dll /r:System.Net.Primitives.dll /r:System.Net.Security.dll /r:System.Security.Cryptography.Primitives.dll /r:System.Text.RegularExpressions.dll ${testdir}/../../resources/stubs/System.Web.cs /r:System.Runtime.Serialization.Primitives.dll + +using System; +using System.IO; +using System.Text; +using System.Collections; +using System.Collections.Generic; +using System.Collections.Specialized; +using System.Linq; +using System.Runtime.Serialization; +using System.Threading.Tasks; +using System.Web; +using System.Web.UI.WebControls; +using System.Text.RegularExpressions; + +public class RegexHandler +{ + private static readonly string JAVA_CLASS_REGEX = "^(([a-z])+.)+[A-Z]([a-z])+$"; + + public void ProcessRequest() + { + string userInput = ""; + + // BAD: + // Artificial regexes + new Regex("^([a-z]+)+$").Match(userInput); + new Regex("^([a-z]*)*$").Replace(userInput, ""); + // Known exponential blowup regex for e-mail address validation + // Problematic part is: ([a-zA-Z0-9]+))* + new Regex("^([a-zA-Z0-9])(([\\-.]|[_]+)?([a-zA-Z0-9]+))*(@){1}[a-z0-9]+[.]{1}(([a-z]{2,3})|([a-z]{2,3}[.]{1}[a-z]{2,3}))$").Match(userInput); + // Known exponential blowup regex for Java class name validation + // Problematic part is: (([a-z])+.)+ + new Regex(JAVA_CLASS_REGEX).Match(userInput); + // Static use + Regex.Match(userInput, JAVA_CLASS_REGEX); + // GOOD: + new Regex("^(([a-b]+[c-z]+)+$").Match(userInput); + new Regex("^([a-z]+)+$", RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1)).Match(userInput); + Regex.Match(userInput, JAVA_CLASS_REGEX, RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1)); + // Known possible FP. + new Regex("^[a-z0-9]+([_.-][a-z0-9]+)*$").Match(userInput); + } +} + +// The only purpose of this class is to make sure the extractor extracts the +// relevant library methods +public class LibraryTypeDataFlow +{ + void M() + { + int i; + int.Parse(""); + int.TryParse("", out i); + + bool b; + bool.Parse(""); + bool.TryParse("", out b); + + Uri uri = null; + uri.ToString(); + + StringReader sr = new StringReader(""); + + string s = new string(new[] { 'a' }); + string.Join("", "", "", ""); + + StringBuilder sb = new StringBuilder(""); + + Lazy l = new Lazy(() => 42); + + IEnumerable ie = null; + ie.GetEnumerator(); + ie.AsParallel(); + ie.AsQueryable(); + IEnumerable ieint = null; + ieint.Select(x => x); + List list = null; + list.Find(x => x > 0); + Stack stack = null; + stack.Peek(); + ArrayList al = null; + ArrayList.FixedSize(al); + SortedList sl = null; + sl.GetByIndex(0); + + Convert.ToInt32("0"); + + DataContract dc = null; + s = dc.AString; + + KeyValuePair kvp = new KeyValuePair(0, ""); + + IEnumerator ienum = null; + object o = ienum.Current; + + IEnumerator ienumint = null; + i = ienumint.Current; + + var task = new Task(() => { }); + Task.WhenAll(null, null); + Task.WhenAny(null, null); + Task.Factory.ContinueWhenAll((Task[])null, (Func)null); + + var task2 = new Task(() => 42); + Task.Factory.ContinueWhenAny(new Task[] { task2 }, t => t.Result.ToString()); + + Encoding.Unicode.GetString(Encoding.Unicode.GetBytes("")); + + Path.Combine("", ""); + Path.GetDirectoryName(""); + Path.GetExtension(""); + Path.GetFileName(""); + Path.GetFileNameWithoutExtension(""); + Path.GetPathRoot(""); + HttpContextBase context = null; + string name = context.Request.QueryString["name"]; + } + + [DataContract] + public class DataContract + { + [DataMember] + public string AString { get; set; } + } +} From 001b9f8b56d77d066da692c497c43057cdc6051e Mon Sep 17 00:00:00 2001 From: yh-semmle Date: Mon, 8 Oct 2018 22:27:39 -0400 Subject: [PATCH 32/58] Java: account for generic exceptions in `java/unreachable-catch-clause` --- .../src/Likely Bugs/Statements/PartiallyMaskedCatch.ql | 5 ++++- .../PartiallyMaskedCatch/PartiallyMaskedCatchTest.java | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/java/ql/src/Likely Bugs/Statements/PartiallyMaskedCatch.ql b/java/ql/src/Likely Bugs/Statements/PartiallyMaskedCatch.ql index ca4bf2744d7..795a5c6f957 100644 --- a/java/ql/src/Likely Bugs/Statements/PartiallyMaskedCatch.ql +++ b/java/ql/src/Likely Bugs/Statements/PartiallyMaskedCatch.ql @@ -47,7 +47,10 @@ RefType getAThrownExceptionType(TryStmt t) { t.getBlock() = call.getEnclosingStmt().getParent*() or t.getAResourceDecl() = call.getEnclosingStmt() | - call.getCallee().getAnException() = e and + ( + call.getCallee().getAnException() = e or + call.(GenericCall).getATypeArgument(call.getCallee().getAnException().getType()) = e.getType() + ) and not caughtInside(t, call.getEnclosingStmt(), e.getType()) and result = e.getType() ) or diff --git a/java/ql/test/query-tests/PartiallyMaskedCatch/PartiallyMaskedCatchTest.java b/java/ql/test/query-tests/PartiallyMaskedCatch/PartiallyMaskedCatchTest.java index 5f07a5e3810..4debe220f25 100644 --- a/java/ql/test/query-tests/PartiallyMaskedCatch/PartiallyMaskedCatchTest.java +++ b/java/ql/test/query-tests/PartiallyMaskedCatch/PartiallyMaskedCatchTest.java @@ -68,6 +68,14 @@ public class PartiallyMaskedCatchTest { } catch (IOException e) { // reachable: IOException is thrown by getClosableThing() } + + try (ClosableThing thing = new ClosableThing()) { + genericThrowingMethod(IOException.class); + } catch (ExceptionA e) { + // reachable: ExceptionA is thrown by implicit invocation of CloseableThing.close() + } catch (IOException e) { + // reachable: IOException is thrown by invocation of genericThrowingMethod(IOException.class) + } } public static ClosableThing getClosableThing() throws IOException { @@ -94,4 +102,6 @@ public class PartiallyMaskedCatchTest { throws ClassNotFoundException, NoSuchMethodException, InstantiationException, IllegalAccessException, InvocationTargetException {} + + public static void genericThrowingMethod(Class c) throws E {} } From 26b630f7005f21e27642430c517697780bdf9748 Mon Sep 17 00:00:00 2001 From: yh-semmle Date: Wed, 19 Sep 2018 16:01:42 -0400 Subject: [PATCH 33/58] Java: clarify help for `java/unreachable-catch-clause` --- .../src/Likely Bugs/Statements/PartiallyMaskedCatch.java | 7 +++---- .../src/Likely Bugs/Statements/PartiallyMaskedCatch.qhelp | 8 +++++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/java/ql/src/Likely Bugs/Statements/PartiallyMaskedCatch.java b/java/ql/src/Likely Bugs/Statements/PartiallyMaskedCatch.java index 4c62a6eb5c8..1ad408b9c17 100644 --- a/java/ql/src/Likely Bugs/Statements/PartiallyMaskedCatch.java +++ b/java/ql/src/Likely Bugs/Statements/PartiallyMaskedCatch.java @@ -1,11 +1,10 @@ -FileInputStream fis = null; +FileOutputStream fos = null; try { - fis = new FileInputStream(new File("may_not_exist.txt")); - // read from input stream + fos = new FileOutputStream(new File("may_not_exist.txt")); } catch (FileNotFoundException e) { // ask the user and try again } catch (IOException e) { // more serious, abort } finally { - if (fis!=null) { try { fis.close(); } catch (IOException e) { /*ignore*/ } } + if (fos!=null) { try { fos.close(); } catch (IOException e) { /*ignore*/ } } } \ No newline at end of file diff --git a/java/ql/src/Likely Bugs/Statements/PartiallyMaskedCatch.qhelp b/java/ql/src/Likely Bugs/Statements/PartiallyMaskedCatch.qhelp index 9e882906e2d..56c81672a21 100644 --- a/java/ql/src/Likely Bugs/Statements/PartiallyMaskedCatch.qhelp +++ b/java/ql/src/Likely Bugs/Statements/PartiallyMaskedCatch.qhelp @@ -45,7 +45,13 @@ than one of the catch clauses, only the first matching clause is ex -

In the following example, the second catch clause is unreachable, and can be removed.

+

+In the following example, the second catch clause is unreachable. +The code is incomplete because a FileOutputStream is opened but +no methods are called to write to the stream. Such methods typically throw +IOExceptions, which would make the second catch clause +reachable. +

From fa3b9a699786c956e58bbda7016660c0f1cbab9f Mon Sep 17 00:00:00 2001 From: yh-semmle Date: Tue, 9 Oct 2018 21:31:19 -0400 Subject: [PATCH 34/58] Java: add change note for `java/unreachable-catch-clause` --- change-notes/1.19/analysis-java.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 change-notes/1.19/analysis-java.md diff --git a/change-notes/1.19/analysis-java.md b/change-notes/1.19/analysis-java.md new file mode 100644 index 00000000000..43b4ef0dfd9 --- /dev/null +++ b/change-notes/1.19/analysis-java.md @@ -0,0 +1,16 @@ +# Improvements to Java analysis + +## General improvements + +## New queries + +| **Query** | **Tags** | **Purpose** | +|-----------------------------------------------|------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| + +## Changes to existing queries + +| **Query** | **Expected impact** | **Change** | +| Unreachable catch clause | Fewer false-positive results | This rule now accounts for calls to generic methods that throw generic exceptions. | + +## Changes to QL libraries + From d261915598f1e5961d484d586234cd48fbae57bb Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Fri, 5 Oct 2018 12:04:21 +0200 Subject: [PATCH 35/58] JS: polish FileAccessToHttp.ql --- .../ql/src/Security/CWE-200/FileAccessToHttp.ql | 8 ++++---- .../Security/CWE-200/FileAccessToHttp.expected | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/javascript/ql/src/Security/CWE-200/FileAccessToHttp.ql b/javascript/ql/src/Security/CWE-200/FileAccessToHttp.ql index dc034abcb24..cfa34da87fd 100644 --- a/javascript/ql/src/Security/CWE-200/FileAccessToHttp.ql +++ b/javascript/ql/src/Security/CWE-200/FileAccessToHttp.ql @@ -1,6 +1,6 @@ /** - * @name File Access data flows to Http POST/PUT - * @description Writing data from file directly to http body or request header can be an indication to data exfiltration or unauthorized information disclosure. + * @name File data in outbound remote request + * @description Directly sending file data in an outbound remote request can indicate unauthorized information disclosure. * @kind problem * @problem.severity warning * @id js/file-access-to-http @@ -11,6 +11,6 @@ import javascript import semmle.javascript.security.dataflow.FileAccessToHttp -from FileAccessToHttpDataFlow::Configuration config, DataFlow::Node src, DataFlow::Node sink +from FileAccessToHttp::Configuration config, DataFlow::Node src, DataFlow::Node sink where config.hasFlow (src, sink) -select src, "$@ flows directly to Http request body", sink, "File access" +select sink, "$@ flows directly to outbound remote request", src, "File data" diff --git a/javascript/ql/test/query-tests/Security/CWE-200/FileAccessToHttp.expected b/javascript/ql/test/query-tests/Security/CWE-200/FileAccessToHttp.expected index 254ae9fc1b4..936c9e0b488 100644 --- a/javascript/ql/test/query-tests/Security/CWE-200/FileAccessToHttp.expected +++ b/javascript/ql/test/query-tests/Security/CWE-200/FileAccessToHttp.expected @@ -1,8 +1,8 @@ -| bufferRead.js:12:22:12:43 | new Buf ... s.size) | $@ flows directly to Http request body | bufferRead.js:33:21:33:28 | postData | File access | -| googlecompiler.js:44:54:44:57 | data | $@ flows directly to Http request body | googlecompiler.js:38:18:38:26 | post_data | File access | -| readFileSync.js:5:12:5:39 | fs.read ... t.txt") | $@ flows directly to Http request body | readFileSync.js:26:18:26:18 | s | File access | -| readStreamRead.js:13:21:13:35 | readable.read() | $@ flows directly to Http request body | readStreamRead.js:30:19:30:23 | chunk | File access | -| request.js:28:52:28:55 | data | $@ flows directly to Http request body | request.js:8:11:8:20 | {jsonData} | File access | -| request.js:43:51:43:54 | data | $@ flows directly to Http request body | request.js:16:11:23:3 | {\\n u ... ody\\n } | File access | -| sentAsHeaders.js:10:79:10:84 | buffer | $@ flows directly to Http request body | sentAsHeaders.js:14:20:19:9 | {\\n ... } | File access | -| sentAsHeaders.js:10:79:10:84 | buffer | $@ flows directly to Http request body | sentAsHeaders.js:20:20:25:9 | {\\n ... } | File access | +| bufferRead.js:33:21:33:28 | postData | $@ flows directly to outbound remote request | bufferRead.js:12:22:12:43 | new Buf ... s.size) | File data | +| googlecompiler.js:38:18:38:26 | post_data | $@ flows directly to outbound remote request | googlecompiler.js:44:54:44:57 | data | File data | +| readFileSync.js:26:18:26:18 | s | $@ flows directly to outbound remote request | readFileSync.js:5:12:5:39 | fs.read ... t.txt") | File data | +| readStreamRead.js:30:19:30:23 | chunk | $@ flows directly to outbound remote request | readStreamRead.js:13:21:13:35 | readable.read() | File data | +| request.js:8:11:8:20 | {jsonData} | $@ flows directly to outbound remote request | request.js:28:52:28:55 | data | File data | +| request.js:16:11:23:3 | {\\n u ... ody\\n } | $@ flows directly to outbound remote request | request.js:43:51:43:54 | data | File data | +| sentAsHeaders.js:14:20:19:9 | {\\n ... } | $@ flows directly to outbound remote request | sentAsHeaders.js:10:79:10:84 | buffer | File data | +| sentAsHeaders.js:20:20:25:9 | {\\n ... } | $@ flows directly to outbound remote request | sentAsHeaders.js:10:79:10:84 | buffer | File data | From b00aa36cdcf3e199cb2ca55208d849a979b74030 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Fri, 5 Oct 2018 12:25:07 +0200 Subject: [PATCH 36/58] JS: polish HttpToFileAccess.ql --- javascript/ql/src/Security/CWE-912/HttpToFileAccess.ql | 8 ++++---- .../Security/CWE-912/HttpToFileAccess.expected | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/javascript/ql/src/Security/CWE-912/HttpToFileAccess.ql b/javascript/ql/src/Security/CWE-912/HttpToFileAccess.ql index 73ba30c2ca2..0b4344fe2e4 100644 --- a/javascript/ql/src/Security/CWE-912/HttpToFileAccess.ql +++ b/javascript/ql/src/Security/CWE-912/HttpToFileAccess.ql @@ -1,6 +1,6 @@ /** - * @name Http response data flows to File Access - * @description Writing data from an HTTP request directly to the file system allows arbitrary file upload and might indicate a backdoor. + * @name User-controlled data in file + * @description Writing user-controlled data directly to the file system allows arbitrary file upload and might indicate a backdoor. * @kind problem * @problem.severity warning * @id js/http-to-file-access @@ -11,6 +11,6 @@ import javascript import semmle.javascript.security.dataflow.HttpToFileAccess -from HttpToFileAccessFlow::Configuration configuration, DataFlow::Node src, DataFlow::Node sink +from HttpToFileAccess::Configuration configuration, DataFlow::Node src, DataFlow::Node sink where configuration.hasFlow(src, sink) -select sink, "$@ flows to file system", src, "Untrusted data received from Http response" +select sink, "$@ flows to file system", src, "Untrusted data" diff --git a/javascript/ql/test/query-tests/Security/CWE-912/HttpToFileAccess.expected b/javascript/ql/test/query-tests/Security/CWE-912/HttpToFileAccess.expected index 326ed804bce..5799343dd26 100644 --- a/javascript/ql/test/query-tests/Security/CWE-912/HttpToFileAccess.expected +++ b/javascript/ql/test/query-tests/Security/CWE-912/HttpToFileAccess.expected @@ -1,3 +1,3 @@ -| tst.js:16:33:16:33 | c | $@ flows to file system | tst.js:15:26:15:26 | c | Untrusted data received from Http response | -| tst.js:19:25:19:25 | c | $@ flows to file system | tst.js:15:26:15:26 | c | Untrusted data received from Http response | -| tst.js:24:22:24:22 | c | $@ flows to file system | tst.js:15:26:15:26 | c | Untrusted data received from Http response | +| tst.js:16:33:16:33 | c | $@ flows to file system | tst.js:15:26:15:26 | c | Untrusted data | +| tst.js:19:25:19:25 | c | $@ flows to file system | tst.js:15:26:15:26 | c | Untrusted data | +| tst.js:24:22:24:22 | c | $@ flows to file system | tst.js:15:26:15:26 | c | Untrusted data | From a3ec739210d00326351f25836d75d6a47043aa79 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Fri, 5 Oct 2018 13:25:53 +0200 Subject: [PATCH 37/58] JS: restructure FileSystemWriteAccess/FileSystemReadAccess API --- .../ql/src/semmle/javascript/Concepts.qll | 20 +++-- .../semmle/javascript/frameworks/Express.qll | 4 - .../javascript/frameworks/NodeJSLib.qll | 89 +++++++++++-------- .../security/dataflow/FileAccessToHttp.qll | 2 +- .../security/dataflow/HttpToFileAccess.qll | 2 +- 5 files changed, 67 insertions(+), 50 deletions(-) diff --git a/javascript/ql/src/semmle/javascript/Concepts.qll b/javascript/ql/src/semmle/javascript/Concepts.qll index 62e48b7f277..dddd27caf14 100644 --- a/javascript/ql/src/semmle/javascript/Concepts.qll +++ b/javascript/ql/src/semmle/javascript/Concepts.qll @@ -29,19 +29,27 @@ abstract class FileSystemAccess extends DataFlow::Node { /** Gets an argument to this file system access that is interpreted as a path. */ abstract DataFlow::Node getAPathArgument(); - /** Gets a node that represents file system access data, such as buffer the data is copied to. */ - abstract DataFlow::Node getDataNode(); } /** - * A data flow node that performs read file system access. + * A data flow node that reads data from the file system. */ -abstract class FileSystemReadAccess extends FileSystemAccess { } +abstract class FileSystemReadAccess extends FileSystemAccess { + + /** Gets a node that represents data from the file system. */ + abstract DataFlow::Node getADataNode(); + +} /** - * A data flow node that performs write file system access. + * A data flow node that writes data to the file system. */ -abstract class FileSystemWriteAccess extends FileSystemAccess { } +abstract class FileSystemWriteAccess extends FileSystemAccess { + + /** Gets a node that represents data to be written to the file system. */ + abstract DataFlow::Node getADataNode(); + +} /** * A data flow node that contains a file name or an array of file names from the local file system. diff --git a/javascript/ql/src/semmle/javascript/frameworks/Express.qll b/javascript/ql/src/semmle/javascript/frameworks/Express.qll index 484ffaf0c49..83662b81611 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/Express.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/Express.qll @@ -832,10 +832,6 @@ module Express { asExpr().(MethodCallExpr).calls(any(ResponseExpr res), name)) } - override DataFlow::Node getDataNode() { - result = DataFlow::valueNode(astNode) - } - override DataFlow::Node getAPathArgument() { result = DataFlow::valueNode(astNode.getArgument(0)) } diff --git a/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll b/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll index 6dbae816ce9..6bd0efc2540 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll @@ -416,26 +416,6 @@ module NodeJSLib { result = methodName } - override DataFlow::Node getDataNode() { - ( - methodName = "readFileSync" and - result = this - ) - or - exists (int i, string paramName | fsDataParam(methodName, i, paramName) | - ( - paramName = "callback" and - exists (DataFlow::ParameterNode p, string n | - p = getCallback(i).getAParameter() and - n = p.getName().toLowerCase() and - result = p | - n = "data" or n = "buffer" or n = "string" - ) - ) - or - result = getArgument(i)) - } - override DataFlow::Node getAPathArgument() { exists (int i | fsFileParam(methodName, i) | result = getArgument(i)) @@ -444,24 +424,57 @@ module NodeJSLib { /** Only NodeJSSystemFileAccessCalls that write data to 'fs' */ private class NodeJSFileSystemAccessWriteCall extends FileSystemWriteAccess, NodeJSFileSystemAccessCall { - NodeJSFileSystemAccessWriteCall () { - this.getMethodName() = "appendFile" or - this.getMethodName() = "appendFileSync" or - this.getMethodName() = "write" or - this.getMethodName() = "writeFile" or - this.getMethodName() = "writeFileSync" or - this.getMethodName() = "writeSync" - } + NodeJSFileSystemAccessWriteCall () { + this.getMethodName() = "appendFile" or + this.getMethodName() = "appendFileSync" or + this.getMethodName() = "write" or + this.getMethodName() = "writeFile" or + this.getMethodName() = "writeFileSync" or + this.getMethodName() = "writeSync" + } + + override DataFlow::Node getADataNode() { + exists (int i, string paramName | + fsDataParam(methodName, i, paramName) | + if paramName = "callback" then + exists (DataFlow::ParameterNode p | + p = getCallback(i).getAParameter() and + p.getName().regexpMatch("(?i)data|buffer|string") and + result = p + ) + else + result = getArgument(i) + ) + } + } /** Only NodeJSSystemFileAccessCalls that read data from 'fs' */ private class NodeJSFileSystemAccessReadCall extends FileSystemReadAccess, NodeJSFileSystemAccessCall { - NodeJSFileSystemAccessReadCall () { - this.getMethodName() = "read" or - this.getMethodName() = "readSync" or - this.getMethodName() = "readFile" or - this.getMethodName() = "readFileSync" - } + NodeJSFileSystemAccessReadCall () { + this.getMethodName() = "read" or + this.getMethodName() = "readSync" or + this.getMethodName() = "readFile" or + this.getMethodName() = "readFileSync" + } + + override DataFlow::Node getADataNode() { + if methodName.regexpMatch(".*Sync") then + result = this + else + exists (int i, string paramName | + fsDataParam(methodName, i, paramName) | + if paramName = "callback" then + exists (DataFlow::ParameterNode p | + p = getCallback(i).getAParameter() and + p.getName().regexpMatch("(?i)data|buffer|string") and + result = p + ) + else + result = getArgument(i) + ) + } + } /** @@ -479,7 +492,7 @@ module NodeJSLib { ) } - override DataFlow::Node getDataNode() { + override DataFlow::Node getADataNode() { result = this.getArgument(0) } @@ -502,7 +515,7 @@ module NodeJSLib { ) } - override DataFlow::Node getDataNode() { + override DataFlow::Node getADataNode() { result = this } @@ -525,7 +538,7 @@ module NodeJSLib { ) } - override DataFlow::Node getDataNode() { + override DataFlow::Node getADataNode() { result = this.getArgument(0) } @@ -549,7 +562,7 @@ module NodeJSLib { ) } - override DataFlow::Node getDataNode() { + override DataFlow::Node getADataNode() { result = this.getCallback(1).getParameter(0) } diff --git a/javascript/ql/src/semmle/javascript/security/dataflow/FileAccessToHttp.qll b/javascript/ql/src/semmle/javascript/security/dataflow/FileAccessToHttp.qll index d73df0ff8e4..a53175426bf 100644 --- a/javascript/ql/src/semmle/javascript/security/dataflow/FileAccessToHttp.qll +++ b/javascript/ql/src/semmle/javascript/security/dataflow/FileAccessToHttp.qll @@ -57,7 +57,7 @@ module FileAccessToHttpDataFlow { private class FileAccessArgumentAsSource extends Source { FileAccessArgumentAsSource() { exists(FileSystemReadAccess src | - this = src.getDataNode().getALocalSource() + this = src.getADataNode().getALocalSource() ) } } diff --git a/javascript/ql/src/semmle/javascript/security/dataflow/HttpToFileAccess.qll b/javascript/ql/src/semmle/javascript/security/dataflow/HttpToFileAccess.qll index 5cc86edf66a..a04f9c1e06a 100644 --- a/javascript/ql/src/semmle/javascript/security/dataflow/HttpToFileAccess.qll +++ b/javascript/ql/src/semmle/javascript/security/dataflow/HttpToFileAccess.qll @@ -49,7 +49,7 @@ module HttpToFileAccessFlow { class FileAccessAsSink extends Sink { FileAccessAsSink () { exists(FileSystemWriteAccess src | - this = src.getDataNode() + this = src.getADataNode() ) } } From 3b2440e8508acb7487eb21493565ec2ccfb87d06 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Fri, 5 Oct 2018 13:41:22 +0200 Subject: [PATCH 38/58] JS: remove useless externs definitions for tests --- .../Security/CWE-200/externs/fs.js | 1699 ----------------- .../Security/CWE-200/fs-read-externs.js | 78 + .../Security/CWE-912/externs/fs.js | 1699 ----------------- .../Security/CWE-912/fs-writeFile-externs.js | 62 + 4 files changed, 140 insertions(+), 3398 deletions(-) delete mode 100644 javascript/ql/test/query-tests/Security/CWE-200/externs/fs.js create mode 100644 javascript/ql/test/query-tests/Security/CWE-200/fs-read-externs.js delete mode 100644 javascript/ql/test/query-tests/Security/CWE-912/externs/fs.js create mode 100644 javascript/ql/test/query-tests/Security/CWE-912/fs-writeFile-externs.js diff --git a/javascript/ql/test/query-tests/Security/CWE-200/externs/fs.js b/javascript/ql/test/query-tests/Security/CWE-200/externs/fs.js deleted file mode 100644 index a1ce1f83a7e..00000000000 --- a/javascript/ql/test/query-tests/Security/CWE-200/externs/fs.js +++ /dev/null @@ -1,1699 +0,0 @@ -// Automatically generated from TypeScript type definitions provided by -// DefinitelyTyped (https://github.com/DefinitelyTyped/DefinitelyTyped), -// which is licensed under the MIT license; see file DefinitelyTyped-LICENSE -// in parent directory. -// Type definitions for Node.js 10.5.x -// Project: http://nodejs.org/ -// Definitions by: Microsoft TypeScript -// DefinitelyTyped -// Parambir Singh -// Christian Vaagland Tellnes -// Wilco Bakker -// Nicolas Voigt -// Chigozirim C. -// Flarna -// Mariusz Wiktorczyk -// wwwy3y3 -// Deividas Bakanas -// Kelvin Jin -// Alvis HT Tang -// Sebastian Silbermann -// Hannes Magnusson -// Alberto Schiabel -// Klaus Meinhardt -// Huw -// Nicolas Even -// Bruno Scheufler -// Mohsen Azimi -// Hoàng Văn Khải -// Alexander T. -// Lishude -// Andrew Makarov -// Zane Hannan AU -// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped - -/** - * @externs - * @fileoverview Definitions for module "fs" - */ - -var fs = {}; - -var events = require("events"); - -/** - * @interface - */ -function Stats() {} - -/** - * @return {boolean} - */ -Stats.prototype.isFile = function() {}; - -/** - * @return {boolean} - */ -Stats.prototype.isDirectory = function() {}; - -/** - * @return {boolean} - */ -Stats.prototype.isBlockDevice = function() {}; - -/** - * @return {boolean} - */ -Stats.prototype.isCharacterDevice = function() {}; - -/** - * @return {boolean} - */ -Stats.prototype.isSymbolicLink = function() {}; - -/** - * @return {boolean} - */ -Stats.prototype.isFIFO = function() {}; - -/** - * @return {boolean} - */ -Stats.prototype.isSocket = function() {}; - -/** - * @type {number} - */ -Stats.prototype.dev; - -/** - * @type {number} - */ -Stats.prototype.ino; - -/** - * @type {number} - */ -Stats.prototype.mode; - -/** - * @type {number} - */ -Stats.prototype.nlink; - -/** - * @type {number} - */ -Stats.prototype.uid; - -/** - * @type {number} - */ -Stats.prototype.gid; - -/** - * @type {number} - */ -Stats.prototype.rdev; - -/** - * @type {number} - */ -Stats.prototype.size; - -/** - * @type {number} - */ -Stats.prototype.blksize; - -/** - * @type {number} - */ -Stats.prototype.blocks; - -/** - * @type {Date} - */ -Stats.prototype.atime; - -/** - * @type {Date} - */ -Stats.prototype.mtime; - -/** - * @type {Date} - */ -Stats.prototype.ctime; - -/** - * @type {Date} - */ -Stats.prototype.birthtime; - -/** - * @interface - * @extends {events.EventEmitter} - */ -function FSWatcher() {} - -/** - * @return {void} - */ -FSWatcher.prototype.close = function() {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -FSWatcher.prototype.addListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(string, (string|Buffer)): void)} listener - * @return {*} - */ -FSWatcher.prototype.addListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number, string): void)} listener - * @return {*} - */ -FSWatcher.prototype.addListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -FSWatcher.prototype.on = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(string, (string|Buffer)): void)} listener - * @return {*} - */ -FSWatcher.prototype.on = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number, string): void)} listener - * @return {*} - */ -FSWatcher.prototype.on = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -FSWatcher.prototype.once = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(string, (string|Buffer)): void)} listener - * @return {*} - */ -FSWatcher.prototype.once = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number, string): void)} listener - * @return {*} - */ -FSWatcher.prototype.once = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -FSWatcher.prototype.prependListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(string, (string|Buffer)): void)} listener - * @return {*} - */ -FSWatcher.prototype.prependListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number, string): void)} listener - * @return {*} - */ -FSWatcher.prototype.prependListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -FSWatcher.prototype.prependOnceListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(string, (string|Buffer)): void)} listener - * @return {*} - */ -FSWatcher.prototype.prependOnceListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number, string): void)} listener - * @return {*} - */ -FSWatcher.prototype.prependOnceListener = function(event, listener) {}; - -/** - * @interface - * @extends {internal.Readable} - */ -fs.ReadStream = function() {}; - -/** - * @return {void} - */ -fs.ReadStream.prototype.close = function() {}; - -/** - * @return {void} - */ -fs.ReadStream.prototype.destroy = function() {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -fs.ReadStream.prototype.addListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number): void)} listener - * @return {*} - */ -fs.ReadStream.prototype.addListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(): void)} listener - * @return {*} - */ -fs.ReadStream.prototype.addListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -fs.ReadStream.prototype.on = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number): void)} listener - * @return {*} - */ -fs.ReadStream.prototype.on = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(): void)} listener - * @return {*} - */ -fs.ReadStream.prototype.on = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -fs.ReadStream.prototype.once = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number): void)} listener - * @return {*} - */ -fs.ReadStream.prototype.once = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(): void)} listener - * @return {*} - */ -fs.ReadStream.prototype.once = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -fs.ReadStream.prototype.prependListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number): void)} listener - * @return {*} - */ -fs.ReadStream.prototype.prependListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(): void)} listener - * @return {*} - */ -fs.ReadStream.prototype.prependListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -fs.ReadStream.prototype.prependOnceListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number): void)} listener - * @return {*} - */ -fs.ReadStream.prototype.prependOnceListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(): void)} listener - * @return {*} - */ -fs.ReadStream.prototype.prependOnceListener = function(event, listener) {}; - -/** - * @interface - * @extends {internal.Writable} - */ -fs.WriteStream = function() {}; - -/** - * @return {void} - */ -fs.WriteStream.prototype.close = function() {}; - -/** - * @type {number} - */ -fs.WriteStream.prototype.bytesWritten; - -/** - * @type {(string|Buffer)} - */ -fs.WriteStream.prototype.path; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -fs.WriteStream.prototype.addListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number): void)} listener - * @return {*} - */ -fs.WriteStream.prototype.addListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(): void)} listener - * @return {*} - */ -fs.WriteStream.prototype.addListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -fs.WriteStream.prototype.on = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number): void)} listener - * @return {*} - */ -fs.WriteStream.prototype.on = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(): void)} listener - * @return {*} - */ -fs.WriteStream.prototype.on = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -fs.WriteStream.prototype.once = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number): void)} listener - * @return {*} - */ -fs.WriteStream.prototype.once = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(): void)} listener - * @return {*} - */ -fs.WriteStream.prototype.once = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -fs.WriteStream.prototype.prependListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number): void)} listener - * @return {*} - */ -fs.WriteStream.prototype.prependListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(): void)} listener - * @return {*} - */ -fs.WriteStream.prototype.prependListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -fs.WriteStream.prototype.prependOnceListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number): void)} listener - * @return {*} - */ -fs.WriteStream.prototype.prependOnceListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(): void)} listener - * @return {*} - */ -fs.WriteStream.prototype.prependOnceListener = function(event, listener) {}; - -/** - * @param {string} oldPath - * @param {string} newPath - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.rename = function(oldPath, newPath, callback) {}; - -/** - * @param {string} oldPath - * @param {string} newPath - * @return {void} - */ -fs.renameSync = function(oldPath, newPath) {}; - -/** - * @param {(string|Buffer)} path - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.truncate = function(path, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} len - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.truncate = function(path, len, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {number=} len - * @return {void} - */ -fs.truncateSync = function(path, len) {}; - -/** - * @param {number} fd - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.ftruncate = function(fd, callback) {}; - -/** - * @param {number} fd - * @param {number} len - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.ftruncate = function(fd, len, callback) {}; - -/** - * @param {number} fd - * @param {number=} len - * @return {void} - */ -fs.ftruncateSync = function(fd, len) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} uid - * @param {number} gid - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.chown = function(path, uid, gid, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} uid - * @param {number} gid - * @return {void} - */ -fs.chownSync = function(path, uid, gid) {}; - -/** - * @param {number} fd - * @param {number} uid - * @param {number} gid - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.fchown = function(fd, uid, gid, callback) {}; - -/** - * @param {number} fd - * @param {number} uid - * @param {number} gid - * @return {void} - */ -fs.fchownSync = function(fd, uid, gid) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} uid - * @param {number} gid - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.lchown = function(path, uid, gid, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} uid - * @param {number} gid - * @return {void} - */ -fs.lchownSync = function(path, uid, gid) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} mode - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.chmod = function(path, mode, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {string} mode - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.chmod = function(path, mode, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} mode - * @return {void} - */ -fs.chmodSync = function(path, mode) {}; - -/** - * @param {(string|Buffer)} path - * @param {string} mode - * @return {void} - */ -fs.chmodSync = function(path, mode) {}; - -/** - * @param {number} fd - * @param {number} mode - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.fchmod = function(fd, mode, callback) {}; - -/** - * @param {number} fd - * @param {string} mode - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.fchmod = function(fd, mode, callback) {}; - -/** - * @param {number} fd - * @param {number} mode - * @return {void} - */ -fs.fchmodSync = function(fd, mode) {}; - -/** - * @param {number} fd - * @param {string} mode - * @return {void} - */ -fs.fchmodSync = function(fd, mode) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} mode - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.lchmod = function(path, mode, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {string} mode - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.lchmod = function(path, mode, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} mode - * @return {void} - */ -fs.lchmodSync = function(path, mode) {}; - -/** - * @param {(string|Buffer)} path - * @param {string} mode - * @return {void} - */ -fs.lchmodSync = function(path, mode) {}; - -/** - * @param {(string|Buffer)} path - * @param {(function(NodeJS.ErrnoException, fs.Stats): *)=} callback - * @return {void} - */ -fs.stat = function(path, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {(function(NodeJS.ErrnoException, fs.Stats): *)=} callback - * @return {void} - */ -fs.lstat = function(path, callback) {}; - -/** - * @param {number} fd - * @param {(function(NodeJS.ErrnoException, fs.Stats): *)=} callback - * @return {void} - */ -fs.fstat = function(fd, callback) {}; - -/** - * @param {(string|Buffer)} path - * @return {fs.Stats} - */ -fs.statSync = function(path) {}; - -/** - * @param {(string|Buffer)} path - * @return {fs.Stats} - */ -fs.lstatSync = function(path) {}; - -/** - * @param {number} fd - * @return {fs.Stats} - */ -fs.fstatSync = function(fd) {}; - -/** - * @param {(string|Buffer)} srcpath - * @param {(string|Buffer)} dstpath - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.link = function(srcpath, dstpath, callback) {}; - -/** - * @param {(string|Buffer)} srcpath - * @param {(string|Buffer)} dstpath - * @return {void} - */ -fs.linkSync = function(srcpath, dstpath) {}; - -/** - * @param {(string|Buffer)} srcpath - * @param {(string|Buffer)} dstpath - * @param {string=} type - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.symlink = function(srcpath, dstpath, type, callback) {}; - -/** - * @param {(string|Buffer)} srcpath - * @param {(string|Buffer)} dstpath - * @param {string=} type - * @return {void} - */ -fs.symlinkSync = function(srcpath, dstpath, type) {}; - -/** - * @param {(string|Buffer)} path - * @param {(function(NodeJS.ErrnoException, string): *)=} callback - * @return {void} - */ -fs.readlink = function(path, callback) {}; - -/** - * @param {(string|Buffer)} path - * @return {string} - */ -fs.readlinkSync = function(path) {}; - -/** - * @param {(string|Buffer)} path - * @param {(function(NodeJS.ErrnoException, string): *)=} callback - * @return {void} - */ -fs.realpath = function(path, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {Object} cache - * @param {(function(NodeJS.ErrnoException, string): *)} callback - * @return {void} - */ -fs.realpath = function(path, cache, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {Object=} cache - * @return {string} - */ -fs.realpathSync = function(path, cache) {}; - -/** - * @param {(string|Buffer)} path - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.unlink = function(path, callback) {}; - -/** - * @param {(string|Buffer)} path - * @return {void} - */ -fs.unlinkSync = function(path) {}; - -/** - * @param {(string|Buffer)} path - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.rmdir = function(path, callback) {}; - -/** - * @param {(string|Buffer)} path - * @return {void} - */ -fs.rmdirSync = function(path) {}; - -/** - * @param {(string|Buffer)} path - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.mkdir = function(path, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} mode - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.mkdir = function(path, mode, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {string} mode - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.mkdir = function(path, mode, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {number=} mode - * @return {void} - */ -fs.mkdirSync = function(path, mode) {}; - -/** - * @param {(string|Buffer)} path - * @param {string=} mode - * @return {void} - */ -fs.mkdirSync = function(path, mode) {}; - -/** - * @param {string} prefix - * @param {(function(NodeJS.ErrnoException, string): void)=} callback - * @return {void} - */ -fs.mkdtemp = function(prefix, callback) {}; - -/** - * @param {string} prefix - * @return {string} - */ -fs.mkdtempSync = function(prefix) {}; - -/** - * @param {(string|Buffer)} path - * @param {(function(NodeJS.ErrnoException, Array): void)=} callback - * @return {void} - */ -fs.readdir = function(path, callback) {}; - -/** - * @param {(string|Buffer)} path - * @return {Array} - */ -fs.readdirSync = function(path) {}; - -/** - * @param {number} fd - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.close = function(fd, callback) {}; - -/** - * @param {number} fd - * @return {void} - */ -fs.closeSync = function(fd) {}; - -/** - * @param {(string|Buffer)} path - * @param {(string|number)} flags - * @param {(function(NodeJS.ErrnoException, number): void)} callback - * @return {void} - */ -fs.open = function(path, flags, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {(string|number)} flags - * @param {number} mode - * @param {(function(NodeJS.ErrnoException, number): void)} callback - * @return {void} - */ -fs.open = function(path, flags, mode, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {(string|number)} flags - * @param {number=} mode - * @return {number} - */ -fs.openSync = function(path, flags, mode) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} atime - * @param {number} mtime - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.utimes = function(path, atime, mtime, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {Date} atime - * @param {Date} mtime - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.utimes = function(path, atime, mtime, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} atime - * @param {number} mtime - * @return {void} - */ -fs.utimesSync = function(path, atime, mtime) {}; - -/** - * @param {(string|Buffer)} path - * @param {Date} atime - * @param {Date} mtime - * @return {void} - */ -fs.utimesSync = function(path, atime, mtime) {}; - -/** - * @param {number} fd - * @param {number} atime - * @param {number} mtime - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.futimes = function(fd, atime, mtime, callback) {}; - -/** - * @param {number} fd - * @param {Date} atime - * @param {Date} mtime - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.futimes = function(fd, atime, mtime, callback) {}; - -/** - * @param {number} fd - * @param {number} atime - * @param {number} mtime - * @return {void} - */ -fs.futimesSync = function(fd, atime, mtime) {}; - -/** - * @param {number} fd - * @param {Date} atime - * @param {Date} mtime - * @return {void} - */ -fs.futimesSync = function(fd, atime, mtime) {}; - -/** - * @param {number} fd - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.fsync = function(fd, callback) {}; - -/** - * @param {number} fd - * @return {void} - */ -fs.fsyncSync = function(fd) {}; - -/** - * @param {number} fd - * @param {Buffer} buffer - * @param {number} offset - * @param {number} length - * @param {number} position - * @param {(function(NodeJS.ErrnoException, number, Buffer): void)=} callback - * @return {void} - */ -fs.write = function(fd, buffer, offset, length, position, callback) {}; - -/** - * @param {number} fd - * @param {Buffer} buffer - * @param {number} offset - * @param {number} length - * @param {(function(NodeJS.ErrnoException, number, Buffer): void)=} callback - * @return {void} - */ -fs.write = function(fd, buffer, offset, length, callback) {}; - -/** - * @param {number} fd - * @param {*} data - * @param {(function(NodeJS.ErrnoException, number, string): void)=} callback - * @return {void} - */ -fs.write = function(fd, data, callback) {}; - -/** - * @param {number} fd - * @param {*} data - * @param {number} offset - * @param {(function(NodeJS.ErrnoException, number, string): void)=} callback - * @return {void} - */ -fs.write = function(fd, data, offset, callback) {}; - -/** - * @param {number} fd - * @param {*} data - * @param {number} offset - * @param {string} encoding - * @param {(function(NodeJS.ErrnoException, number, string): void)=} callback - * @return {void} - */ -fs.write = function(fd, data, offset, encoding, callback) {}; - -/** - * @param {number} fd - * @param {Buffer} buffer - * @param {number} offset - * @param {number} length - * @param {number=} position - * @return {number} - */ -fs.writeSync = function(fd, buffer, offset, length, position) {}; - -/** - * @param {number} fd - * @param {*} data - * @param {number=} position - * @param {string=} enconding - * @return {number} - */ -fs.writeSync = function(fd, data, position, enconding) {}; - -/** - * @param {number} fd - * @param {Buffer} buffer - * @param {number} offset - * @param {number} length - * @param {number} position - * @param {(function(NodeJS.ErrnoException, number, Buffer): void)=} callback - * @return {void} - */ -fs.read = function(fd, buffer, offset, length, position, callback) {}; - -/** - * @param {number} fd - * @param {Buffer} buffer - * @param {number} offset - * @param {number} length - * @param {number} position - * @return {number} - */ -fs.readSync = function(fd, buffer, offset, length, position) {}; - -/** - * @param {string} filename - * @param {string} encoding - * @param {(function(NodeJS.ErrnoException, string): void)} callback - * @return {void} - */ -fs.readFile = function(filename, encoding, callback) {}; - -/** - * @param {string} filename - * @param {{encoding: string, flag: string}} options - * @param {(function(NodeJS.ErrnoException, string): void)} callback - * @return {void} - */ -fs.readFile = function(filename, options, callback) {}; - -/** - * @param {string} filename - * @param {{flag: string}} options - * @param {(function(NodeJS.ErrnoException, Buffer): void)} callback - * @return {void} - */ -fs.readFile = function(filename, options, callback) {}; - -/** - * @param {string} filename - * @param {(function(NodeJS.ErrnoException, Buffer): void)} callback - * @return {void} - */ -fs.readFile = function(filename, callback) {}; - -/** - * @param {string} filename - * @param {string} encoding - * @return {string} - */ -fs.readFileSync = function(filename, encoding) {}; - -/** - * @param {string} filename - * @param {{encoding: string, flag: string}} options - * @return {string} - */ -fs.readFileSync = function(filename, options) {}; - -/** - * @param {string} filename - * @param {{flag: string}=} options - * @return {Buffer} - */ -fs.readFileSync = function(filename, options) {}; - -/** - * @param {string} filename - * @param {*} data - * @param {(function(NodeJS.ErrnoException): void)=} callback - * @return {void} - */ -fs.writeFile = function(filename, data, callback) {}; - -/** - * @param {string} filename - * @param {*} data - * @param {{encoding: string, mode: number, flag: string}} options - * @param {(function(NodeJS.ErrnoException): void)=} callback - * @return {void} - */ -fs.writeFile = function(filename, data, options, callback) {}; - -/** - * @param {string} filename - * @param {*} data - * @param {{encoding: string, mode: string, flag: string}} options - * @param {(function(NodeJS.ErrnoException): void)=} callback - * @return {void} - */ -fs.writeFile = function(filename, data, options, callback) {}; - -/** - * @param {string} filename - * @param {*} data - * @param {{encoding: string, mode: number, flag: string}=} options - * @return {void} - */ -fs.writeFileSync = function(filename, data, options) {}; - -/** - * @param {string} filename - * @param {*} data - * @param {{encoding: string, mode: string, flag: string}=} options - * @return {void} - */ -fs.writeFileSync = function(filename, data, options) {}; - -/** - * @param {string} filename - * @param {*} data - * @param {{encoding: string, mode: number, flag: string}} options - * @param {(function(NodeJS.ErrnoException): void)=} callback - * @return {void} - */ -fs.appendFile = function(filename, data, options, callback) {}; - -/** - * @param {string} filename - * @param {*} data - * @param {{encoding: string, mode: string, flag: string}} options - * @param {(function(NodeJS.ErrnoException): void)=} callback - * @return {void} - */ -fs.appendFile = function(filename, data, options, callback) {}; - -/** - * @param {string} filename - * @param {*} data - * @param {(function(NodeJS.ErrnoException): void)=} callback - * @return {void} - */ -fs.appendFile = function(filename, data, callback) {}; - -/** - * @param {string} filename - * @param {*} data - * @param {{encoding: string, mode: number, flag: string}=} options - * @return {void} - */ -fs.appendFileSync = function(filename, data, options) {}; - -/** - * @param {string} filename - * @param {*} data - * @param {{encoding: string, mode: string, flag: string}=} options - * @return {void} - */ -fs.appendFileSync = function(filename, data, options) {}; - -/** - * @param {string} filename - * @param {(function(fs.Stats, fs.Stats): void)} listener - * @return {void} - */ -fs.watchFile = function(filename, listener) {}; - -/** - * @param {string} filename - * @param {{persistent: boolean, interval: number}} options - * @param {(function(fs.Stats, fs.Stats): void)} listener - * @return {void} - */ -fs.watchFile = function(filename, options, listener) {}; - -/** - * @param {string} filename - * @param {(function(fs.Stats, fs.Stats): void)=} listener - * @return {void} - */ -fs.unwatchFile = function(filename, listener) {}; - -/** - * @param {string} filename - * @param {(function(string, string): *)=} listener - * @return {fs.FSWatcher} - */ -fs.watch = function(filename, listener) {}; - -/** - * @param {string} filename - * @param {string} encoding - * @param {(function(string, (string|Buffer)): *)=} listener - * @return {fs.FSWatcher} - */ -fs.watch = function(filename, encoding, listener) {}; - -/** - * @param {string} filename - * @param {{persistent: boolean, recursive: boolean, encoding: string}} options - * @param {(function(string, (string|Buffer)): *)=} listener - * @return {fs.FSWatcher} - */ -fs.watch = function(filename, options, listener) {}; - -/** - * @param {(string|Buffer)} path - * @param {(function(boolean): void)=} callback - * @return {void} - */ -fs.exists = function(path, callback) {}; - -/** - * @param {(string|Buffer)} path - * @return {boolean} - */ -fs.existsSync = function(path) {}; - -/** - * @interface - */ -function Constants() {} - -/** - * @type {number} - */ -Constants.prototype.F_OK; - -/** - * @type {number} - */ -Constants.prototype.R_OK; - -/** - * @type {number} - */ -Constants.prototype.W_OK; - -/** - * @type {number} - */ -Constants.prototype.X_OK; - -/** - * @type {fs.Constants} - */ -fs.constants; - -/** - * @param {(string|Buffer)} path - * @param {(function(NodeJS.ErrnoException): void)} callback - * @return {void} - */ -fs.access = function(path, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} mode - * @param {(function(NodeJS.ErrnoException): void)} callback - * @return {void} - */ -fs.access = function(path, mode, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {number=} mode - * @return {void} - */ -fs.accessSync = function(path, mode) {}; - -/** - * @param {(string|Buffer)} path - * @param {{flags: string, encoding: string, fd: number, mode: number, autoClose: boolean, start: number, end: number}=} options - * @return {fs.ReadStream} - */ -fs.createReadStream = function(path, options) {}; - -/** - * @param {(string|Buffer)} path - * @param {{flags: string, encoding: string, fd: number, mode: number}=} options - * @return {fs.WriteStream} - */ -fs.createWriteStream = function(path, options) {}; - -/** - * @param {number} fd - * @param {Function} callback - * @return {void} - */ -fs.fdatasync = function(fd, callback) {}; - -/** - * @param {number} fd - * @return {void} - */ -fs.fdatasyncSync = function(fd) {}; - -module.exports.ReadStream = fs.ReadStream; - -module.exports.WriteStream = fs.WriteStream; - -module.exports.rename = fs.rename; - -module.exports.renameSync = fs.renameSync; - -module.exports.truncate = fs.truncate; - -module.exports.truncate = fs.truncate; - -module.exports.truncateSync = fs.truncateSync; - -module.exports.ftruncate = fs.ftruncate; - -module.exports.ftruncate = fs.ftruncate; - -module.exports.ftruncateSync = fs.ftruncateSync; - -module.exports.chown = fs.chown; - -module.exports.chownSync = fs.chownSync; - -module.exports.fchown = fs.fchown; - -module.exports.fchownSync = fs.fchownSync; - -module.exports.lchown = fs.lchown; - -module.exports.lchownSync = fs.lchownSync; - -module.exports.chmod = fs.chmod; - -module.exports.chmod = fs.chmod; - -module.exports.chmodSync = fs.chmodSync; - -module.exports.chmodSync = fs.chmodSync; - -module.exports.fchmod = fs.fchmod; - -module.exports.fchmod = fs.fchmod; - -module.exports.fchmodSync = fs.fchmodSync; - -module.exports.fchmodSync = fs.fchmodSync; - -module.exports.lchmod = fs.lchmod; - -module.exports.lchmod = fs.lchmod; - -module.exports.lchmodSync = fs.lchmodSync; - -module.exports.lchmodSync = fs.lchmodSync; - -module.exports.stat = fs.stat; - -module.exports.lstat = fs.lstat; - -module.exports.fstat = fs.fstat; - -module.exports.statSync = fs.statSync; - -module.exports.lstatSync = fs.lstatSync; - -module.exports.fstatSync = fs.fstatSync; - -module.exports.link = fs.link; - -module.exports.linkSync = fs.linkSync; - -module.exports.symlink = fs.symlink; - -module.exports.symlinkSync = fs.symlinkSync; - -module.exports.readlink = fs.readlink; - -module.exports.readlinkSync = fs.readlinkSync; - -module.exports.realpath = fs.realpath; - -module.exports.realpath = fs.realpath; - -module.exports.realpathSync = fs.realpathSync; - -module.exports.unlink = fs.unlink; - -module.exports.unlinkSync = fs.unlinkSync; - -module.exports.rmdir = fs.rmdir; - -module.exports.rmdirSync = fs.rmdirSync; - -module.exports.mkdir = fs.mkdir; - -module.exports.mkdir = fs.mkdir; - -module.exports.mkdir = fs.mkdir; - -module.exports.mkdirSync = fs.mkdirSync; - -module.exports.mkdirSync = fs.mkdirSync; - -module.exports.mkdtemp = fs.mkdtemp; - -module.exports.mkdtempSync = fs.mkdtempSync; - -module.exports.readdir = fs.readdir; - -module.exports.readdirSync = fs.readdirSync; - -module.exports.close = fs.close; - -module.exports.closeSync = fs.closeSync; - -module.exports.open = fs.open; - -module.exports.open = fs.open; - -module.exports.openSync = fs.openSync; - -module.exports.utimes = fs.utimes; - -module.exports.utimes = fs.utimes; - -module.exports.utimesSync = fs.utimesSync; - -module.exports.utimesSync = fs.utimesSync; - -module.exports.futimes = fs.futimes; - -module.exports.futimes = fs.futimes; - -module.exports.futimesSync = fs.futimesSync; - -module.exports.futimesSync = fs.futimesSync; - -module.exports.fsync = fs.fsync; - -module.exports.fsyncSync = fs.fsyncSync; - -module.exports.write = fs.write; - -module.exports.write = fs.write; - -module.exports.write = fs.write; - -module.exports.write = fs.write; - -module.exports.write = fs.write; - -module.exports.writeSync = fs.writeSync; - -module.exports.writeSync = fs.writeSync; - -module.exports.read = fs.read; - -module.exports.readSync = fs.readSync; - -module.exports.readFile = fs.readFile; - -module.exports.readFile = fs.readFile; - -module.exports.readFile = fs.readFile; - -module.exports.readFile = fs.readFile; - -module.exports.readFileSync = fs.readFileSync; - -module.exports.readFileSync = fs.readFileSync; - -module.exports.readFileSync = fs.readFileSync; - -module.exports.writeFile = fs.writeFile; - -module.exports.writeFile = fs.writeFile; - -module.exports.writeFile = fs.writeFile; - -module.exports.writeFileSync = fs.writeFileSync; - -module.exports.writeFileSync = fs.writeFileSync; - -module.exports.appendFile = fs.appendFile; - -module.exports.appendFile = fs.appendFile; - -module.exports.appendFile = fs.appendFile; - -module.exports.appendFileSync = fs.appendFileSync; - -module.exports.appendFileSync = fs.appendFileSync; - -module.exports.watchFile = fs.watchFile; - -module.exports.watchFile = fs.watchFile; - -module.exports.unwatchFile = fs.unwatchFile; - -module.exports.watch = fs.watch; - -module.exports.watch = fs.watch; - -module.exports.watch = fs.watch; - -module.exports.exists = fs.exists; - -module.exports.existsSync = fs.existsSync; - -module.exports.constants = fs.constants; - -module.exports.access = fs.access; - -module.exports.access = fs.access; - -module.exports.accessSync = fs.accessSync; - -module.exports.createReadStream = fs.createReadStream; - -module.exports.createWriteStream = fs.createWriteStream; - -module.exports.fdatasync = fs.fdatasync; - -module.exports.fdatasyncSync = fs.fdatasyncSync; - -/** - * @param {string} path - * @param {(number|Date)} atime - * @param {(number|Date)} mtime - * @param {number=} flags - * @param {Function=} callback - * @return {void} - */ -fs.utimensat = function(path, atime, mtime, flags, callback) {}; - -/** - * @param {string} path - * @param {(number|Date)} atime - * @param {(number|Date)} mtime - * @param {number=} flags - * @return {void} - */ -fs.utimensatSync = function(path, atime, mtime, flags) {}; - -/** - * @param {*} fd - * @param {(number|Date)} atime - * @param {(number|Date)} mtime - * @param {number=} flags - * @param {Function=} callback - * @return {void} - */ -fs.futimensat = function(fd, atime, mtime, flags, callback) {}; - -/** - * @param {*} fd - * @param {(number|Date)} atime - * @param {(number|Date)} mtime - * @param {number=} flags - * @return {void} - */ -fs.futimensatSync = function(fd, atime, mtime, flags) {}; - -/** - * @constructor - * @extends {internal.Writable} - */ -fs.SyncWriteStream; - -/** - * @type {number} - */ -fs.F_OK; - -/** - * @type {number} - */ -fs.R_OK; - -/** - * @type {number} - */ -fs.W_OK; - -/** - * @type {number} - */ -fs.X_OK; - -module.exports.utimensat = fs.utimensat; - -module.exports.utimensatSync = fs.utimensatSync; - -module.exports.futimensat = fs.futimensat; - -module.exports.futimensatSync = fs.futimensatSync; - -module.exports.SyncWriteStream = fs.SyncWriteStream; - -module.exports.F_OK = fs.F_OK; - -module.exports.R_OK = fs.R_OK; - -module.exports.W_OK = fs.W_OK; - -module.exports.X_OK = fs.X_OK; - diff --git a/javascript/ql/test/query-tests/Security/CWE-200/fs-read-externs.js b/javascript/ql/test/query-tests/Security/CWE-200/fs-read-externs.js new file mode 100644 index 00000000000..4d5e9c370b5 --- /dev/null +++ b/javascript/ql/test/query-tests/Security/CWE-200/fs-read-externs.js @@ -0,0 +1,78 @@ +// Automatically generated from TypeScript type definitions provided by +// DefinitelyTyped (https://github.com/DefinitelyTyped/DefinitelyTyped), +// which is licensed under the MIT license; see file DefinitelyTyped-LICENSE +// in parent directory. +// Type definitions for Node.js 10.5.x +// Project: http://nodejs.org/ +// Definitions by: Microsoft TypeScript +// DefinitelyTyped +// Parambir Singh +// Christian Vaagland Tellnes +// Wilco Bakker +// Nicolas Voigt +// Chigozirim C. +// Flarna +// Mariusz Wiktorczyk +// wwwy3y3 +// Deividas Bakanas +// Kelvin Jin +// Alvis HT Tang +// Sebastian Silbermann +// Hannes Magnusson +// Alberto Schiabel +// Klaus Meinhardt +// Huw +// Nicolas Even +// Bruno Scheufler +// Mohsen Azimi +// Hoàng Văn Khải +// Alexander T. +// Lishude +// Andrew Makarov +// Zane Hannan AU +// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped + +/** + * @externs + * @fileoverview Definitions for module "fs" + */ +var fs = {}; + +/** + * @param {number} fd + * @param {Buffer} buffer + * @param {number} offset + * @param {number} length + * @param {number} position + * @param {(function(NodeJS.ErrnoException, number, Buffer): void)=} callback + * @return {void} + */ +fs.read = function(fd, buffer, offset, length, position, callback) {}; + +/** + * @param {string} filename + * @param {string} encoding + * @param {(function(NodeJS.ErrnoException, string): void)} callback + * @return {void} + */ +fs.readFile = function(filename, encoding, callback) {}; +/** + * @param {string} filename + * @param {{encoding: string, flag: string}} options + * @param {(function(NodeJS.ErrnoException, string): void)} callback + * @return {void} + */ +fs.readFile = function(filename, options, callback) {}; +/** + * @param {string} filename + * @param {{flag: string}} options + * @param {(function(NodeJS.ErrnoException, Buffer): void)} callback + * @return {void} + */ +fs.readFile = function(filename, options, callback) {}; +/** + * @param {string} filename + * @param {(function(NodeJS.ErrnoException, Buffer): void)} callback + * @return {void} + */ +fs.readFile = function(filename, callback) {}; diff --git a/javascript/ql/test/query-tests/Security/CWE-912/externs/fs.js b/javascript/ql/test/query-tests/Security/CWE-912/externs/fs.js deleted file mode 100644 index a1ce1f83a7e..00000000000 --- a/javascript/ql/test/query-tests/Security/CWE-912/externs/fs.js +++ /dev/null @@ -1,1699 +0,0 @@ -// Automatically generated from TypeScript type definitions provided by -// DefinitelyTyped (https://github.com/DefinitelyTyped/DefinitelyTyped), -// which is licensed under the MIT license; see file DefinitelyTyped-LICENSE -// in parent directory. -// Type definitions for Node.js 10.5.x -// Project: http://nodejs.org/ -// Definitions by: Microsoft TypeScript -// DefinitelyTyped -// Parambir Singh -// Christian Vaagland Tellnes -// Wilco Bakker -// Nicolas Voigt -// Chigozirim C. -// Flarna -// Mariusz Wiktorczyk -// wwwy3y3 -// Deividas Bakanas -// Kelvin Jin -// Alvis HT Tang -// Sebastian Silbermann -// Hannes Magnusson -// Alberto Schiabel -// Klaus Meinhardt -// Huw -// Nicolas Even -// Bruno Scheufler -// Mohsen Azimi -// Hoàng Văn Khải -// Alexander T. -// Lishude -// Andrew Makarov -// Zane Hannan AU -// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped - -/** - * @externs - * @fileoverview Definitions for module "fs" - */ - -var fs = {}; - -var events = require("events"); - -/** - * @interface - */ -function Stats() {} - -/** - * @return {boolean} - */ -Stats.prototype.isFile = function() {}; - -/** - * @return {boolean} - */ -Stats.prototype.isDirectory = function() {}; - -/** - * @return {boolean} - */ -Stats.prototype.isBlockDevice = function() {}; - -/** - * @return {boolean} - */ -Stats.prototype.isCharacterDevice = function() {}; - -/** - * @return {boolean} - */ -Stats.prototype.isSymbolicLink = function() {}; - -/** - * @return {boolean} - */ -Stats.prototype.isFIFO = function() {}; - -/** - * @return {boolean} - */ -Stats.prototype.isSocket = function() {}; - -/** - * @type {number} - */ -Stats.prototype.dev; - -/** - * @type {number} - */ -Stats.prototype.ino; - -/** - * @type {number} - */ -Stats.prototype.mode; - -/** - * @type {number} - */ -Stats.prototype.nlink; - -/** - * @type {number} - */ -Stats.prototype.uid; - -/** - * @type {number} - */ -Stats.prototype.gid; - -/** - * @type {number} - */ -Stats.prototype.rdev; - -/** - * @type {number} - */ -Stats.prototype.size; - -/** - * @type {number} - */ -Stats.prototype.blksize; - -/** - * @type {number} - */ -Stats.prototype.blocks; - -/** - * @type {Date} - */ -Stats.prototype.atime; - -/** - * @type {Date} - */ -Stats.prototype.mtime; - -/** - * @type {Date} - */ -Stats.prototype.ctime; - -/** - * @type {Date} - */ -Stats.prototype.birthtime; - -/** - * @interface - * @extends {events.EventEmitter} - */ -function FSWatcher() {} - -/** - * @return {void} - */ -FSWatcher.prototype.close = function() {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -FSWatcher.prototype.addListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(string, (string|Buffer)): void)} listener - * @return {*} - */ -FSWatcher.prototype.addListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number, string): void)} listener - * @return {*} - */ -FSWatcher.prototype.addListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -FSWatcher.prototype.on = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(string, (string|Buffer)): void)} listener - * @return {*} - */ -FSWatcher.prototype.on = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number, string): void)} listener - * @return {*} - */ -FSWatcher.prototype.on = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -FSWatcher.prototype.once = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(string, (string|Buffer)): void)} listener - * @return {*} - */ -FSWatcher.prototype.once = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number, string): void)} listener - * @return {*} - */ -FSWatcher.prototype.once = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -FSWatcher.prototype.prependListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(string, (string|Buffer)): void)} listener - * @return {*} - */ -FSWatcher.prototype.prependListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number, string): void)} listener - * @return {*} - */ -FSWatcher.prototype.prependListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -FSWatcher.prototype.prependOnceListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(string, (string|Buffer)): void)} listener - * @return {*} - */ -FSWatcher.prototype.prependOnceListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number, string): void)} listener - * @return {*} - */ -FSWatcher.prototype.prependOnceListener = function(event, listener) {}; - -/** - * @interface - * @extends {internal.Readable} - */ -fs.ReadStream = function() {}; - -/** - * @return {void} - */ -fs.ReadStream.prototype.close = function() {}; - -/** - * @return {void} - */ -fs.ReadStream.prototype.destroy = function() {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -fs.ReadStream.prototype.addListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number): void)} listener - * @return {*} - */ -fs.ReadStream.prototype.addListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(): void)} listener - * @return {*} - */ -fs.ReadStream.prototype.addListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -fs.ReadStream.prototype.on = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number): void)} listener - * @return {*} - */ -fs.ReadStream.prototype.on = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(): void)} listener - * @return {*} - */ -fs.ReadStream.prototype.on = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -fs.ReadStream.prototype.once = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number): void)} listener - * @return {*} - */ -fs.ReadStream.prototype.once = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(): void)} listener - * @return {*} - */ -fs.ReadStream.prototype.once = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -fs.ReadStream.prototype.prependListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number): void)} listener - * @return {*} - */ -fs.ReadStream.prototype.prependListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(): void)} listener - * @return {*} - */ -fs.ReadStream.prototype.prependListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -fs.ReadStream.prototype.prependOnceListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number): void)} listener - * @return {*} - */ -fs.ReadStream.prototype.prependOnceListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(): void)} listener - * @return {*} - */ -fs.ReadStream.prototype.prependOnceListener = function(event, listener) {}; - -/** - * @interface - * @extends {internal.Writable} - */ -fs.WriteStream = function() {}; - -/** - * @return {void} - */ -fs.WriteStream.prototype.close = function() {}; - -/** - * @type {number} - */ -fs.WriteStream.prototype.bytesWritten; - -/** - * @type {(string|Buffer)} - */ -fs.WriteStream.prototype.path; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -fs.WriteStream.prototype.addListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number): void)} listener - * @return {*} - */ -fs.WriteStream.prototype.addListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(): void)} listener - * @return {*} - */ -fs.WriteStream.prototype.addListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -fs.WriteStream.prototype.on = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number): void)} listener - * @return {*} - */ -fs.WriteStream.prototype.on = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(): void)} listener - * @return {*} - */ -fs.WriteStream.prototype.on = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -fs.WriteStream.prototype.once = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number): void)} listener - * @return {*} - */ -fs.WriteStream.prototype.once = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(): void)} listener - * @return {*} - */ -fs.WriteStream.prototype.once = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -fs.WriteStream.prototype.prependListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number): void)} listener - * @return {*} - */ -fs.WriteStream.prototype.prependListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(): void)} listener - * @return {*} - */ -fs.WriteStream.prototype.prependListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {Function} listener - * @return {*} - */ -fs.WriteStream.prototype.prependOnceListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(number): void)} listener - * @return {*} - */ -fs.WriteStream.prototype.prependOnceListener = function(event, listener) {}; - -/** - * @param {string} event - * @param {(function(): void)} listener - * @return {*} - */ -fs.WriteStream.prototype.prependOnceListener = function(event, listener) {}; - -/** - * @param {string} oldPath - * @param {string} newPath - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.rename = function(oldPath, newPath, callback) {}; - -/** - * @param {string} oldPath - * @param {string} newPath - * @return {void} - */ -fs.renameSync = function(oldPath, newPath) {}; - -/** - * @param {(string|Buffer)} path - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.truncate = function(path, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} len - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.truncate = function(path, len, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {number=} len - * @return {void} - */ -fs.truncateSync = function(path, len) {}; - -/** - * @param {number} fd - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.ftruncate = function(fd, callback) {}; - -/** - * @param {number} fd - * @param {number} len - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.ftruncate = function(fd, len, callback) {}; - -/** - * @param {number} fd - * @param {number=} len - * @return {void} - */ -fs.ftruncateSync = function(fd, len) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} uid - * @param {number} gid - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.chown = function(path, uid, gid, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} uid - * @param {number} gid - * @return {void} - */ -fs.chownSync = function(path, uid, gid) {}; - -/** - * @param {number} fd - * @param {number} uid - * @param {number} gid - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.fchown = function(fd, uid, gid, callback) {}; - -/** - * @param {number} fd - * @param {number} uid - * @param {number} gid - * @return {void} - */ -fs.fchownSync = function(fd, uid, gid) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} uid - * @param {number} gid - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.lchown = function(path, uid, gid, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} uid - * @param {number} gid - * @return {void} - */ -fs.lchownSync = function(path, uid, gid) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} mode - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.chmod = function(path, mode, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {string} mode - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.chmod = function(path, mode, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} mode - * @return {void} - */ -fs.chmodSync = function(path, mode) {}; - -/** - * @param {(string|Buffer)} path - * @param {string} mode - * @return {void} - */ -fs.chmodSync = function(path, mode) {}; - -/** - * @param {number} fd - * @param {number} mode - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.fchmod = function(fd, mode, callback) {}; - -/** - * @param {number} fd - * @param {string} mode - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.fchmod = function(fd, mode, callback) {}; - -/** - * @param {number} fd - * @param {number} mode - * @return {void} - */ -fs.fchmodSync = function(fd, mode) {}; - -/** - * @param {number} fd - * @param {string} mode - * @return {void} - */ -fs.fchmodSync = function(fd, mode) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} mode - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.lchmod = function(path, mode, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {string} mode - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.lchmod = function(path, mode, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} mode - * @return {void} - */ -fs.lchmodSync = function(path, mode) {}; - -/** - * @param {(string|Buffer)} path - * @param {string} mode - * @return {void} - */ -fs.lchmodSync = function(path, mode) {}; - -/** - * @param {(string|Buffer)} path - * @param {(function(NodeJS.ErrnoException, fs.Stats): *)=} callback - * @return {void} - */ -fs.stat = function(path, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {(function(NodeJS.ErrnoException, fs.Stats): *)=} callback - * @return {void} - */ -fs.lstat = function(path, callback) {}; - -/** - * @param {number} fd - * @param {(function(NodeJS.ErrnoException, fs.Stats): *)=} callback - * @return {void} - */ -fs.fstat = function(fd, callback) {}; - -/** - * @param {(string|Buffer)} path - * @return {fs.Stats} - */ -fs.statSync = function(path) {}; - -/** - * @param {(string|Buffer)} path - * @return {fs.Stats} - */ -fs.lstatSync = function(path) {}; - -/** - * @param {number} fd - * @return {fs.Stats} - */ -fs.fstatSync = function(fd) {}; - -/** - * @param {(string|Buffer)} srcpath - * @param {(string|Buffer)} dstpath - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.link = function(srcpath, dstpath, callback) {}; - -/** - * @param {(string|Buffer)} srcpath - * @param {(string|Buffer)} dstpath - * @return {void} - */ -fs.linkSync = function(srcpath, dstpath) {}; - -/** - * @param {(string|Buffer)} srcpath - * @param {(string|Buffer)} dstpath - * @param {string=} type - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.symlink = function(srcpath, dstpath, type, callback) {}; - -/** - * @param {(string|Buffer)} srcpath - * @param {(string|Buffer)} dstpath - * @param {string=} type - * @return {void} - */ -fs.symlinkSync = function(srcpath, dstpath, type) {}; - -/** - * @param {(string|Buffer)} path - * @param {(function(NodeJS.ErrnoException, string): *)=} callback - * @return {void} - */ -fs.readlink = function(path, callback) {}; - -/** - * @param {(string|Buffer)} path - * @return {string} - */ -fs.readlinkSync = function(path) {}; - -/** - * @param {(string|Buffer)} path - * @param {(function(NodeJS.ErrnoException, string): *)=} callback - * @return {void} - */ -fs.realpath = function(path, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {Object} cache - * @param {(function(NodeJS.ErrnoException, string): *)} callback - * @return {void} - */ -fs.realpath = function(path, cache, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {Object=} cache - * @return {string} - */ -fs.realpathSync = function(path, cache) {}; - -/** - * @param {(string|Buffer)} path - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.unlink = function(path, callback) {}; - -/** - * @param {(string|Buffer)} path - * @return {void} - */ -fs.unlinkSync = function(path) {}; - -/** - * @param {(string|Buffer)} path - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.rmdir = function(path, callback) {}; - -/** - * @param {(string|Buffer)} path - * @return {void} - */ -fs.rmdirSync = function(path) {}; - -/** - * @param {(string|Buffer)} path - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.mkdir = function(path, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} mode - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.mkdir = function(path, mode, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {string} mode - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.mkdir = function(path, mode, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {number=} mode - * @return {void} - */ -fs.mkdirSync = function(path, mode) {}; - -/** - * @param {(string|Buffer)} path - * @param {string=} mode - * @return {void} - */ -fs.mkdirSync = function(path, mode) {}; - -/** - * @param {string} prefix - * @param {(function(NodeJS.ErrnoException, string): void)=} callback - * @return {void} - */ -fs.mkdtemp = function(prefix, callback) {}; - -/** - * @param {string} prefix - * @return {string} - */ -fs.mkdtempSync = function(prefix) {}; - -/** - * @param {(string|Buffer)} path - * @param {(function(NodeJS.ErrnoException, Array): void)=} callback - * @return {void} - */ -fs.readdir = function(path, callback) {}; - -/** - * @param {(string|Buffer)} path - * @return {Array} - */ -fs.readdirSync = function(path) {}; - -/** - * @param {number} fd - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.close = function(fd, callback) {}; - -/** - * @param {number} fd - * @return {void} - */ -fs.closeSync = function(fd) {}; - -/** - * @param {(string|Buffer)} path - * @param {(string|number)} flags - * @param {(function(NodeJS.ErrnoException, number): void)} callback - * @return {void} - */ -fs.open = function(path, flags, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {(string|number)} flags - * @param {number} mode - * @param {(function(NodeJS.ErrnoException, number): void)} callback - * @return {void} - */ -fs.open = function(path, flags, mode, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {(string|number)} flags - * @param {number=} mode - * @return {number} - */ -fs.openSync = function(path, flags, mode) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} atime - * @param {number} mtime - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.utimes = function(path, atime, mtime, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {Date} atime - * @param {Date} mtime - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.utimes = function(path, atime, mtime, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} atime - * @param {number} mtime - * @return {void} - */ -fs.utimesSync = function(path, atime, mtime) {}; - -/** - * @param {(string|Buffer)} path - * @param {Date} atime - * @param {Date} mtime - * @return {void} - */ -fs.utimesSync = function(path, atime, mtime) {}; - -/** - * @param {number} fd - * @param {number} atime - * @param {number} mtime - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.futimes = function(fd, atime, mtime, callback) {}; - -/** - * @param {number} fd - * @param {Date} atime - * @param {Date} mtime - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.futimes = function(fd, atime, mtime, callback) {}; - -/** - * @param {number} fd - * @param {number} atime - * @param {number} mtime - * @return {void} - */ -fs.futimesSync = function(fd, atime, mtime) {}; - -/** - * @param {number} fd - * @param {Date} atime - * @param {Date} mtime - * @return {void} - */ -fs.futimesSync = function(fd, atime, mtime) {}; - -/** - * @param {number} fd - * @param {(function(NodeJS.ErrnoException=): void)=} callback - * @return {void} - */ -fs.fsync = function(fd, callback) {}; - -/** - * @param {number} fd - * @return {void} - */ -fs.fsyncSync = function(fd) {}; - -/** - * @param {number} fd - * @param {Buffer} buffer - * @param {number} offset - * @param {number} length - * @param {number} position - * @param {(function(NodeJS.ErrnoException, number, Buffer): void)=} callback - * @return {void} - */ -fs.write = function(fd, buffer, offset, length, position, callback) {}; - -/** - * @param {number} fd - * @param {Buffer} buffer - * @param {number} offset - * @param {number} length - * @param {(function(NodeJS.ErrnoException, number, Buffer): void)=} callback - * @return {void} - */ -fs.write = function(fd, buffer, offset, length, callback) {}; - -/** - * @param {number} fd - * @param {*} data - * @param {(function(NodeJS.ErrnoException, number, string): void)=} callback - * @return {void} - */ -fs.write = function(fd, data, callback) {}; - -/** - * @param {number} fd - * @param {*} data - * @param {number} offset - * @param {(function(NodeJS.ErrnoException, number, string): void)=} callback - * @return {void} - */ -fs.write = function(fd, data, offset, callback) {}; - -/** - * @param {number} fd - * @param {*} data - * @param {number} offset - * @param {string} encoding - * @param {(function(NodeJS.ErrnoException, number, string): void)=} callback - * @return {void} - */ -fs.write = function(fd, data, offset, encoding, callback) {}; - -/** - * @param {number} fd - * @param {Buffer} buffer - * @param {number} offset - * @param {number} length - * @param {number=} position - * @return {number} - */ -fs.writeSync = function(fd, buffer, offset, length, position) {}; - -/** - * @param {number} fd - * @param {*} data - * @param {number=} position - * @param {string=} enconding - * @return {number} - */ -fs.writeSync = function(fd, data, position, enconding) {}; - -/** - * @param {number} fd - * @param {Buffer} buffer - * @param {number} offset - * @param {number} length - * @param {number} position - * @param {(function(NodeJS.ErrnoException, number, Buffer): void)=} callback - * @return {void} - */ -fs.read = function(fd, buffer, offset, length, position, callback) {}; - -/** - * @param {number} fd - * @param {Buffer} buffer - * @param {number} offset - * @param {number} length - * @param {number} position - * @return {number} - */ -fs.readSync = function(fd, buffer, offset, length, position) {}; - -/** - * @param {string} filename - * @param {string} encoding - * @param {(function(NodeJS.ErrnoException, string): void)} callback - * @return {void} - */ -fs.readFile = function(filename, encoding, callback) {}; - -/** - * @param {string} filename - * @param {{encoding: string, flag: string}} options - * @param {(function(NodeJS.ErrnoException, string): void)} callback - * @return {void} - */ -fs.readFile = function(filename, options, callback) {}; - -/** - * @param {string} filename - * @param {{flag: string}} options - * @param {(function(NodeJS.ErrnoException, Buffer): void)} callback - * @return {void} - */ -fs.readFile = function(filename, options, callback) {}; - -/** - * @param {string} filename - * @param {(function(NodeJS.ErrnoException, Buffer): void)} callback - * @return {void} - */ -fs.readFile = function(filename, callback) {}; - -/** - * @param {string} filename - * @param {string} encoding - * @return {string} - */ -fs.readFileSync = function(filename, encoding) {}; - -/** - * @param {string} filename - * @param {{encoding: string, flag: string}} options - * @return {string} - */ -fs.readFileSync = function(filename, options) {}; - -/** - * @param {string} filename - * @param {{flag: string}=} options - * @return {Buffer} - */ -fs.readFileSync = function(filename, options) {}; - -/** - * @param {string} filename - * @param {*} data - * @param {(function(NodeJS.ErrnoException): void)=} callback - * @return {void} - */ -fs.writeFile = function(filename, data, callback) {}; - -/** - * @param {string} filename - * @param {*} data - * @param {{encoding: string, mode: number, flag: string}} options - * @param {(function(NodeJS.ErrnoException): void)=} callback - * @return {void} - */ -fs.writeFile = function(filename, data, options, callback) {}; - -/** - * @param {string} filename - * @param {*} data - * @param {{encoding: string, mode: string, flag: string}} options - * @param {(function(NodeJS.ErrnoException): void)=} callback - * @return {void} - */ -fs.writeFile = function(filename, data, options, callback) {}; - -/** - * @param {string} filename - * @param {*} data - * @param {{encoding: string, mode: number, flag: string}=} options - * @return {void} - */ -fs.writeFileSync = function(filename, data, options) {}; - -/** - * @param {string} filename - * @param {*} data - * @param {{encoding: string, mode: string, flag: string}=} options - * @return {void} - */ -fs.writeFileSync = function(filename, data, options) {}; - -/** - * @param {string} filename - * @param {*} data - * @param {{encoding: string, mode: number, flag: string}} options - * @param {(function(NodeJS.ErrnoException): void)=} callback - * @return {void} - */ -fs.appendFile = function(filename, data, options, callback) {}; - -/** - * @param {string} filename - * @param {*} data - * @param {{encoding: string, mode: string, flag: string}} options - * @param {(function(NodeJS.ErrnoException): void)=} callback - * @return {void} - */ -fs.appendFile = function(filename, data, options, callback) {}; - -/** - * @param {string} filename - * @param {*} data - * @param {(function(NodeJS.ErrnoException): void)=} callback - * @return {void} - */ -fs.appendFile = function(filename, data, callback) {}; - -/** - * @param {string} filename - * @param {*} data - * @param {{encoding: string, mode: number, flag: string}=} options - * @return {void} - */ -fs.appendFileSync = function(filename, data, options) {}; - -/** - * @param {string} filename - * @param {*} data - * @param {{encoding: string, mode: string, flag: string}=} options - * @return {void} - */ -fs.appendFileSync = function(filename, data, options) {}; - -/** - * @param {string} filename - * @param {(function(fs.Stats, fs.Stats): void)} listener - * @return {void} - */ -fs.watchFile = function(filename, listener) {}; - -/** - * @param {string} filename - * @param {{persistent: boolean, interval: number}} options - * @param {(function(fs.Stats, fs.Stats): void)} listener - * @return {void} - */ -fs.watchFile = function(filename, options, listener) {}; - -/** - * @param {string} filename - * @param {(function(fs.Stats, fs.Stats): void)=} listener - * @return {void} - */ -fs.unwatchFile = function(filename, listener) {}; - -/** - * @param {string} filename - * @param {(function(string, string): *)=} listener - * @return {fs.FSWatcher} - */ -fs.watch = function(filename, listener) {}; - -/** - * @param {string} filename - * @param {string} encoding - * @param {(function(string, (string|Buffer)): *)=} listener - * @return {fs.FSWatcher} - */ -fs.watch = function(filename, encoding, listener) {}; - -/** - * @param {string} filename - * @param {{persistent: boolean, recursive: boolean, encoding: string}} options - * @param {(function(string, (string|Buffer)): *)=} listener - * @return {fs.FSWatcher} - */ -fs.watch = function(filename, options, listener) {}; - -/** - * @param {(string|Buffer)} path - * @param {(function(boolean): void)=} callback - * @return {void} - */ -fs.exists = function(path, callback) {}; - -/** - * @param {(string|Buffer)} path - * @return {boolean} - */ -fs.existsSync = function(path) {}; - -/** - * @interface - */ -function Constants() {} - -/** - * @type {number} - */ -Constants.prototype.F_OK; - -/** - * @type {number} - */ -Constants.prototype.R_OK; - -/** - * @type {number} - */ -Constants.prototype.W_OK; - -/** - * @type {number} - */ -Constants.prototype.X_OK; - -/** - * @type {fs.Constants} - */ -fs.constants; - -/** - * @param {(string|Buffer)} path - * @param {(function(NodeJS.ErrnoException): void)} callback - * @return {void} - */ -fs.access = function(path, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {number} mode - * @param {(function(NodeJS.ErrnoException): void)} callback - * @return {void} - */ -fs.access = function(path, mode, callback) {}; - -/** - * @param {(string|Buffer)} path - * @param {number=} mode - * @return {void} - */ -fs.accessSync = function(path, mode) {}; - -/** - * @param {(string|Buffer)} path - * @param {{flags: string, encoding: string, fd: number, mode: number, autoClose: boolean, start: number, end: number}=} options - * @return {fs.ReadStream} - */ -fs.createReadStream = function(path, options) {}; - -/** - * @param {(string|Buffer)} path - * @param {{flags: string, encoding: string, fd: number, mode: number}=} options - * @return {fs.WriteStream} - */ -fs.createWriteStream = function(path, options) {}; - -/** - * @param {number} fd - * @param {Function} callback - * @return {void} - */ -fs.fdatasync = function(fd, callback) {}; - -/** - * @param {number} fd - * @return {void} - */ -fs.fdatasyncSync = function(fd) {}; - -module.exports.ReadStream = fs.ReadStream; - -module.exports.WriteStream = fs.WriteStream; - -module.exports.rename = fs.rename; - -module.exports.renameSync = fs.renameSync; - -module.exports.truncate = fs.truncate; - -module.exports.truncate = fs.truncate; - -module.exports.truncateSync = fs.truncateSync; - -module.exports.ftruncate = fs.ftruncate; - -module.exports.ftruncate = fs.ftruncate; - -module.exports.ftruncateSync = fs.ftruncateSync; - -module.exports.chown = fs.chown; - -module.exports.chownSync = fs.chownSync; - -module.exports.fchown = fs.fchown; - -module.exports.fchownSync = fs.fchownSync; - -module.exports.lchown = fs.lchown; - -module.exports.lchownSync = fs.lchownSync; - -module.exports.chmod = fs.chmod; - -module.exports.chmod = fs.chmod; - -module.exports.chmodSync = fs.chmodSync; - -module.exports.chmodSync = fs.chmodSync; - -module.exports.fchmod = fs.fchmod; - -module.exports.fchmod = fs.fchmod; - -module.exports.fchmodSync = fs.fchmodSync; - -module.exports.fchmodSync = fs.fchmodSync; - -module.exports.lchmod = fs.lchmod; - -module.exports.lchmod = fs.lchmod; - -module.exports.lchmodSync = fs.lchmodSync; - -module.exports.lchmodSync = fs.lchmodSync; - -module.exports.stat = fs.stat; - -module.exports.lstat = fs.lstat; - -module.exports.fstat = fs.fstat; - -module.exports.statSync = fs.statSync; - -module.exports.lstatSync = fs.lstatSync; - -module.exports.fstatSync = fs.fstatSync; - -module.exports.link = fs.link; - -module.exports.linkSync = fs.linkSync; - -module.exports.symlink = fs.symlink; - -module.exports.symlinkSync = fs.symlinkSync; - -module.exports.readlink = fs.readlink; - -module.exports.readlinkSync = fs.readlinkSync; - -module.exports.realpath = fs.realpath; - -module.exports.realpath = fs.realpath; - -module.exports.realpathSync = fs.realpathSync; - -module.exports.unlink = fs.unlink; - -module.exports.unlinkSync = fs.unlinkSync; - -module.exports.rmdir = fs.rmdir; - -module.exports.rmdirSync = fs.rmdirSync; - -module.exports.mkdir = fs.mkdir; - -module.exports.mkdir = fs.mkdir; - -module.exports.mkdir = fs.mkdir; - -module.exports.mkdirSync = fs.mkdirSync; - -module.exports.mkdirSync = fs.mkdirSync; - -module.exports.mkdtemp = fs.mkdtemp; - -module.exports.mkdtempSync = fs.mkdtempSync; - -module.exports.readdir = fs.readdir; - -module.exports.readdirSync = fs.readdirSync; - -module.exports.close = fs.close; - -module.exports.closeSync = fs.closeSync; - -module.exports.open = fs.open; - -module.exports.open = fs.open; - -module.exports.openSync = fs.openSync; - -module.exports.utimes = fs.utimes; - -module.exports.utimes = fs.utimes; - -module.exports.utimesSync = fs.utimesSync; - -module.exports.utimesSync = fs.utimesSync; - -module.exports.futimes = fs.futimes; - -module.exports.futimes = fs.futimes; - -module.exports.futimesSync = fs.futimesSync; - -module.exports.futimesSync = fs.futimesSync; - -module.exports.fsync = fs.fsync; - -module.exports.fsyncSync = fs.fsyncSync; - -module.exports.write = fs.write; - -module.exports.write = fs.write; - -module.exports.write = fs.write; - -module.exports.write = fs.write; - -module.exports.write = fs.write; - -module.exports.writeSync = fs.writeSync; - -module.exports.writeSync = fs.writeSync; - -module.exports.read = fs.read; - -module.exports.readSync = fs.readSync; - -module.exports.readFile = fs.readFile; - -module.exports.readFile = fs.readFile; - -module.exports.readFile = fs.readFile; - -module.exports.readFile = fs.readFile; - -module.exports.readFileSync = fs.readFileSync; - -module.exports.readFileSync = fs.readFileSync; - -module.exports.readFileSync = fs.readFileSync; - -module.exports.writeFile = fs.writeFile; - -module.exports.writeFile = fs.writeFile; - -module.exports.writeFile = fs.writeFile; - -module.exports.writeFileSync = fs.writeFileSync; - -module.exports.writeFileSync = fs.writeFileSync; - -module.exports.appendFile = fs.appendFile; - -module.exports.appendFile = fs.appendFile; - -module.exports.appendFile = fs.appendFile; - -module.exports.appendFileSync = fs.appendFileSync; - -module.exports.appendFileSync = fs.appendFileSync; - -module.exports.watchFile = fs.watchFile; - -module.exports.watchFile = fs.watchFile; - -module.exports.unwatchFile = fs.unwatchFile; - -module.exports.watch = fs.watch; - -module.exports.watch = fs.watch; - -module.exports.watch = fs.watch; - -module.exports.exists = fs.exists; - -module.exports.existsSync = fs.existsSync; - -module.exports.constants = fs.constants; - -module.exports.access = fs.access; - -module.exports.access = fs.access; - -module.exports.accessSync = fs.accessSync; - -module.exports.createReadStream = fs.createReadStream; - -module.exports.createWriteStream = fs.createWriteStream; - -module.exports.fdatasync = fs.fdatasync; - -module.exports.fdatasyncSync = fs.fdatasyncSync; - -/** - * @param {string} path - * @param {(number|Date)} atime - * @param {(number|Date)} mtime - * @param {number=} flags - * @param {Function=} callback - * @return {void} - */ -fs.utimensat = function(path, atime, mtime, flags, callback) {}; - -/** - * @param {string} path - * @param {(number|Date)} atime - * @param {(number|Date)} mtime - * @param {number=} flags - * @return {void} - */ -fs.utimensatSync = function(path, atime, mtime, flags) {}; - -/** - * @param {*} fd - * @param {(number|Date)} atime - * @param {(number|Date)} mtime - * @param {number=} flags - * @param {Function=} callback - * @return {void} - */ -fs.futimensat = function(fd, atime, mtime, flags, callback) {}; - -/** - * @param {*} fd - * @param {(number|Date)} atime - * @param {(number|Date)} mtime - * @param {number=} flags - * @return {void} - */ -fs.futimensatSync = function(fd, atime, mtime, flags) {}; - -/** - * @constructor - * @extends {internal.Writable} - */ -fs.SyncWriteStream; - -/** - * @type {number} - */ -fs.F_OK; - -/** - * @type {number} - */ -fs.R_OK; - -/** - * @type {number} - */ -fs.W_OK; - -/** - * @type {number} - */ -fs.X_OK; - -module.exports.utimensat = fs.utimensat; - -module.exports.utimensatSync = fs.utimensatSync; - -module.exports.futimensat = fs.futimensat; - -module.exports.futimensatSync = fs.futimensatSync; - -module.exports.SyncWriteStream = fs.SyncWriteStream; - -module.exports.F_OK = fs.F_OK; - -module.exports.R_OK = fs.R_OK; - -module.exports.W_OK = fs.W_OK; - -module.exports.X_OK = fs.X_OK; - diff --git a/javascript/ql/test/query-tests/Security/CWE-912/fs-writeFile-externs.js b/javascript/ql/test/query-tests/Security/CWE-912/fs-writeFile-externs.js new file mode 100644 index 00000000000..93059a72cdd --- /dev/null +++ b/javascript/ql/test/query-tests/Security/CWE-912/fs-writeFile-externs.js @@ -0,0 +1,62 @@ +// Automatically generated from TypeScript type definitions provided by +// DefinitelyTyped (https://github.com/DefinitelyTyped/DefinitelyTyped), +// which is licensed under the MIT license; see file DefinitelyTyped-LICENSE +// in parent directory. +// Type definitions for Node.js 10.5.x +// Project: http://nodejs.org/ +// Definitions by: Microsoft TypeScript +// DefinitelyTyped +// Parambir Singh +// Christian Vaagland Tellnes +// Wilco Bakker +// Nicolas Voigt +// Chigozirim C. +// Flarna +// Mariusz Wiktorczyk +// wwwy3y3 +// Deividas Bakanas +// Kelvin Jin +// Alvis HT Tang +// Sebastian Silbermann +// Hannes Magnusson +// Alberto Schiabel +// Klaus Meinhardt +// Huw +// Nicolas Even +// Bruno Scheufler +// Mohsen Azimi +// Hoàng Văn Khải +// Alexander T. +// Lishude +// Andrew Makarov +// Zane Hannan AU +// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped + /** + * @externs + * @fileoverview Definitions for module "fs" + */ + var fs = {}; + +/** + * @param {string} filename + * @param {*} data + * @param {(function(NodeJS.ErrnoException): void)=} callback + * @return {void} + */ +fs.writeFile = function(filename, data, callback) {}; + /** + * @param {string} filename + * @param {*} data + * @param {{encoding: string, mode: number, flag: string}} options + * @param {(function(NodeJS.ErrnoException): void)=} callback + * @return {void} + */ +fs.writeFile = function(filename, data, options, callback) {}; + /** + * @param {string} filename + * @param {*} data + * @param {{encoding: string, mode: string, flag: string}} options + * @param {(function(NodeJS.ErrnoException): void)=} callback + * @return {void} + */ +fs.writeFile = function(filename, data, options, callback) {}; From 0fc56e443e1dff05595b66079b4f31bda96036bf Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Fri, 5 Oct 2018 14:17:08 +0200 Subject: [PATCH 39/58] JS: introduce ClientRequest.getADataNode --- .../javascript/frameworks/ClientRequests.qll | 42 +++++++++++++++++++ .../semmle/javascript/frameworks/Electron.qll | 8 ++++ .../javascript/frameworks/NodeJSLib.qll | 4 ++ 3 files changed, 54 insertions(+) diff --git a/javascript/ql/src/semmle/javascript/frameworks/ClientRequests.qll b/javascript/ql/src/semmle/javascript/frameworks/ClientRequests.qll index a19b5effca3..bb83e14c3c3 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/ClientRequests.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/ClientRequests.qll @@ -9,6 +9,10 @@ import javascript /** * A call that performs a request to a URL. + * + * Example: An HTTP POST request is client request that sends some + * `data` to a `url`, where both the headers and the body of the request + * contribute to the `data`. */ abstract class CustomClientRequest extends DataFlow::InvokeNode { @@ -16,10 +20,20 @@ abstract class CustomClientRequest extends DataFlow::InvokeNode { * Gets the URL of the request. */ abstract DataFlow::Node getUrl(); + + /** + * Gets a node that contributes to the data-part this request. + */ + abstract DataFlow::Node getADataNode(); + } /** * A call that performs a request to a URL. + * + * Example: An HTTP POST request is client request that sends some + * `data` to a `url`, where both the headers and the body of the request + * contribute to the `data`. */ class ClientRequest extends DataFlow::InvokeNode { @@ -35,6 +49,14 @@ class ClientRequest extends DataFlow::InvokeNode { DataFlow::Node getUrl() { result = custom.getUrl() } + + /** + * Gets a node that contributes to the data-part this request. + */ + DataFlow::Node getADataNode() { + result = custom.getADataNode() + } + } /** @@ -83,6 +105,10 @@ private class RequestUrlRequest extends CustomClientRequest { result = url } + override DataFlow::Node getADataNode() { + none() + } + } /** @@ -113,6 +139,10 @@ private class AxiosUrlRequest extends CustomClientRequest { result = url } + override DataFlow::Node getADataNode() { + none() + } + } /** @@ -144,6 +174,10 @@ private class FetchUrlRequest extends CustomClientRequest { result = url } + override DataFlow::Node getADataNode() { + none() + } + } /** @@ -169,6 +203,10 @@ private class GotUrlRequest extends CustomClientRequest { result = url } + override DataFlow::Node getADataNode() { + none() + } + } /** @@ -191,4 +229,8 @@ private class SuperAgentUrlRequest extends CustomClientRequest { result = url } + override DataFlow::Node getADataNode() { + none() + } + } diff --git a/javascript/ql/src/semmle/javascript/frameworks/Electron.qll b/javascript/ql/src/semmle/javascript/frameworks/Electron.qll index b7c80eca803..bafdd5f2ffb 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/Electron.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/Electron.qll @@ -63,6 +63,10 @@ module Electron { result = getOptionArgument(0, "url") } + override DataFlow::Node getADataNode() { + none() + } + } /** @@ -78,6 +82,10 @@ module Electron { result = getOptionArgument(0, "url") } + override DataFlow::Node getADataNode() { + none() + } + } diff --git a/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll b/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll index 6bd0efc2540..072f5772026 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll @@ -774,6 +774,10 @@ module NodeJSLib { result = url } + override DataFlow::Node getADataNode() { + none() + } + } /** From 4e4597a24d1ff3fcdfe647b078573a2a4b27f92a Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Fri, 5 Oct 2018 14:36:32 +0200 Subject: [PATCH 40/58] JS: replace HTTP::RequestBody with ClientRequest.getADataNode --- .../javascript/frameworks/ClientRequests.qll | 2 +- .../ql/src/semmle/javascript/frameworks/HTTP.qll | 5 ----- .../src/semmle/javascript/frameworks/NodeJSLib.qll | 14 +------------- .../src/semmle/javascript/frameworks/Request.qll | 8 -------- .../security/dataflow/FileAccessToHttp.qll | 13 +++++++++---- 5 files changed, 11 insertions(+), 31 deletions(-) diff --git a/javascript/ql/src/semmle/javascript/frameworks/ClientRequests.qll b/javascript/ql/src/semmle/javascript/frameworks/ClientRequests.qll index bb83e14c3c3..470b533790b 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/ClientRequests.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/ClientRequests.qll @@ -106,7 +106,7 @@ private class RequestUrlRequest extends CustomClientRequest { } override DataFlow::Node getADataNode() { - none() + result = getArgument(1) } } diff --git a/javascript/ql/src/semmle/javascript/frameworks/HTTP.qll b/javascript/ql/src/semmle/javascript/frameworks/HTTP.qll index 169e1140206..d0294f1e512 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/HTTP.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/HTTP.qll @@ -132,11 +132,6 @@ module HTTP { result = "http" or result = "https" } - /** - * An expression whose value is sent as (part of) the body of an HTTP request (POST, PUT). - */ - abstract class RequestBody extends DataFlow::Node {} - /** * An expression whose value is sent as (part of) the body of an HTTP response. */ diff --git a/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll b/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll index 072f5772026..2b43b356660 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll @@ -775,7 +775,7 @@ module NodeJSLib { } override DataFlow::Node getADataNode() { - none() + result = getAMethodCall("write").getArgument(0) } } @@ -811,18 +811,6 @@ module NodeJSLib { result = "http.request data parameter" } } - - /** - * An argument to client request.write () method, can be used to write body to a HTTP or HTTPS POST/PUT request, - * or request option (like headers, cookies, even url) - */ - class HttpRequestWriteArgument extends HTTP::RequestBody, DataFlow::Node { - HttpRequestWriteArgument () { - exists(CustomClientRequest req | - this = req.getAMethodCall("write").getArgument(0) or - this = req.getArgument(0)) - } - } /** * A data flow node that is registered as a callback for an HTTP or HTTPS request made by a Node.js process, for example the function `handler` in `http.request(url).on(message, handler)`. diff --git a/javascript/ql/src/semmle/javascript/frameworks/Request.qll b/javascript/ql/src/semmle/javascript/frameworks/Request.qll index 5416e989997..45406863aba 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/Request.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/Request.qll @@ -44,13 +44,5 @@ module Request { } } - - // using 'request' library to make http 'POST' and 'PUT' requests with message body. - private class RequestPostBody extends HTTP::RequestBody { - RequestPostBody () { - this = DataFlow::moduleMember("request", "post").getACall().getArgument(1) or - this = DataFlow::moduleImport("request").getAnInvocation().getArgument(0) - } - } } \ No newline at end of file diff --git a/javascript/ql/src/semmle/javascript/security/dataflow/FileAccessToHttp.qll b/javascript/ql/src/semmle/javascript/security/dataflow/FileAccessToHttp.qll index a53175426bf..754fd7ded8b 100644 --- a/javascript/ql/src/semmle/javascript/security/dataflow/FileAccessToHttp.qll +++ b/javascript/ql/src/semmle/javascript/security/dataflow/FileAccessToHttp.qll @@ -62,10 +62,15 @@ module FileAccessToHttpDataFlow { } } - /** Sink is any parameter or argument that evaluates to a parameter ot a function or call that sets Http Body on a request */ - private class HttpRequestBodyAsSink extends Sink { - HttpRequestBodyAsSink () { - this instanceof HTTP::RequestBody + /** + * The URL or data of a client request, viewed as a sink. + */ + private class ClientRequestUrlOrDataAsSink extends Sink { + ClientRequestUrlOrDataAsSink () { + exists (ClientRequest req | + this = req.getUrl() or + this = req.getADataNode() + ) } } } \ No newline at end of file From e99b9d34c538f3a02b01931a3a6c58a500cdae37 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Fri, 5 Oct 2018 14:50:20 +0200 Subject: [PATCH 41/58] JS: polish characters of NodeJSFileSystemAccess*Call --- .../javascript/frameworks/NodeJSLib.qll | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll b/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll index 2b43b356660..54fda3e1d37 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll @@ -425,12 +425,12 @@ module NodeJSLib { /** Only NodeJSSystemFileAccessCalls that write data to 'fs' */ private class NodeJSFileSystemAccessWriteCall extends FileSystemWriteAccess, NodeJSFileSystemAccessCall { NodeJSFileSystemAccessWriteCall () { - this.getMethodName() = "appendFile" or - this.getMethodName() = "appendFileSync" or - this.getMethodName() = "write" or - this.getMethodName() = "writeFile" or - this.getMethodName() = "writeFileSync" or - this.getMethodName() = "writeSync" + methodName = "appendFile" or + methodName = "appendFileSync" or + methodName = "write" or + methodName = "writeFile" or + methodName = "writeFileSync" or + methodName = "writeSync" } override DataFlow::Node getADataNode() { @@ -452,10 +452,10 @@ module NodeJSLib { /** Only NodeJSSystemFileAccessCalls that read data from 'fs' */ private class NodeJSFileSystemAccessReadCall extends FileSystemReadAccess, NodeJSFileSystemAccessCall { NodeJSFileSystemAccessReadCall () { - this.getMethodName() = "read" or - this.getMethodName() = "readSync" or - this.getMethodName() = "readFile" or - this.getMethodName() = "readFileSync" + methodName = "read" or + methodName = "readSync" or + methodName = "readFile" or + methodName = "readFileSync" } override DataFlow::Node getADataNode() { From 30f7f41dff1b4151b00841b57a2b1dc56aed4f9f Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Fri, 5 Oct 2018 15:03:09 +0200 Subject: [PATCH 42/58] JS: refactor NodeJSFileSystemWrite to FileStreamWrite --- .../javascript/frameworks/NodeJSLib.qll | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll b/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll index 54fda3e1d37..03ec9d0f409 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll @@ -478,29 +478,30 @@ module NodeJSLib { } /** - * A call to write corresponds to a pattern where file stream is open first with 'createWriteStream', followed by 'write' or 'end' call + * A write to the file system using a stream. */ - private class NodeJSFileSystemWrite extends FileSystemWriteAccess, DataFlow::CallNode { + private class FileStreamWrite extends FileSystemWriteAccess, DataFlow::CallNode { - NodeJSFileSystemAccessCall init; + NodeJSFileSystemAccessCall stream; - NodeJSFileSystemWrite() { - exists (NodeJSFileSystemAccessCall n | - n.getCalleeName() = "createWriteStream" and init = n | - this = n.getAMemberCall("write") or - this = n.getAMemberCall("end") - ) - } - - override DataFlow::Node getADataNode() { - result = this.getArgument(0) - } + FileStreamWrite() { + stream.getMethodName() = "createWriteStream" and + exists (string method | + method = "write" or + method = "end" | + this = stream.getAMemberCall(method) + ) + } + + override DataFlow::Node getADataNode() { + result = getArgument(0) + } + + override DataFlow::Node getAPathArgument() { + result = stream.getAPathArgument() + } + } - override DataFlow::Node getAPathArgument() { - result = init.getAPathArgument() - } - } - /** * A call to read corresponds to a pattern where file stream is open first with createReadStream, followed by 'read' call */ From 43f98a7ef85c5896c8494612b840196af3a92158 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Fri, 5 Oct 2018 15:14:37 +0200 Subject: [PATCH 43/58] JS: refactor NodeJSFileSystemRead* to FileStreamRead --- .../javascript/frameworks/NodeJSLib.qll | 76 +++++-------------- 1 file changed, 19 insertions(+), 57 deletions(-) diff --git a/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll b/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll index 03ec9d0f409..232e7e2564d 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll @@ -503,72 +503,34 @@ module NodeJSLib { } /** - * A call to read corresponds to a pattern where file stream is open first with createReadStream, followed by 'read' call + * A read from the file system using a stream. */ - private class NodeJSFileSystemRead extends FileSystemReadAccess, DataFlow::CallNode { + private class FileStreamRead extends FileSystemReadAccess, DataFlow::CallNode { - NodeJSFileSystemAccessCall init; + NodeJSFileSystemAccessCall stream; - NodeJSFileSystemRead() { - exists (NodeJSFileSystemAccessCall n | - n.getCalleeName() = "createReadStream" and init = n | - this = n.getAMemberCall("read") - ) - } + string method; - override DataFlow::Node getADataNode() { - result = this - } - - override DataFlow::Node getAPathArgument() { - result = init.getAPathArgument() - } - } - - /** - * A call to read corresponds to a pattern where file stream is open first with createReadStream, followed by 'pipe' call - */ - private class NodeJSFileSystemPipe extends FileSystemReadAccess, DataFlow::CallNode { - - NodeJSFileSystemAccessCall init; - - NodeJSFileSystemPipe() { - exists (NodeJSFileSystemAccessCall n | - n.getCalleeName() = "createReadStream" and init = n | - this = n.getAMemberCall("pipe") - ) - } - - override DataFlow::Node getADataNode() { - result = this.getArgument(0) - } - - override DataFlow::Node getAPathArgument() { - result = init.getAPathArgument() - } - } - - /** - * An 'on' event where data comes in as a parameter (usage: readstream.on('data', chunk)) - */ - private class NodeJSFileSystemReadDataEvent extends FileSystemReadAccess, DataFlow::CallNode { - - NodeJSFileSystemAccessCall init; - - NodeJSFileSystemReadDataEvent() { - exists(NodeJSFileSystemAccessCall n | - n.getCalleeName() = "createReadStream" and init = n | - this = n.getAMethodCall("on") and - this.getArgument(0).mayHaveStringValue("data") - ) + FileStreamRead() { + stream.getMethodName() = "createReadStream" and + this = stream.getAMemberCall(method) and + (method = "read" or method = "pipe" or method = "on") } override DataFlow::Node getADataNode() { - result = this.getCallback(1).getParameter(0) + method = "read" and + result = this + or + method = "pipe" and + result = getArgument(0) + or + method = "on" and + getArgument(0).mayHaveStringValue("data") and + result = getCallback(1).getParameter(0) } - + override DataFlow::Node getAPathArgument() { - result = init.getAPathArgument() + result = stream.getAPathArgument() } } From df72492f167e91ae18253408c5674c0cc846f257 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Fri, 5 Oct 2018 15:27:58 +0200 Subject: [PATCH 44/58] JS: polish FileAccessToHttp.qll --- .../security/dataflow/FileAccessToHttp.qll | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/javascript/ql/src/semmle/javascript/security/dataflow/FileAccessToHttp.qll b/javascript/ql/src/semmle/javascript/security/dataflow/FileAccessToHttp.qll index 754fd7ded8b..42a92ce7ef9 100644 --- a/javascript/ql/src/semmle/javascript/security/dataflow/FileAccessToHttp.qll +++ b/javascript/ql/src/semmle/javascript/security/dataflow/FileAccessToHttp.qll @@ -1,30 +1,33 @@ -/** - * Provides Taint tracking configuration for reasoning about file access taint flow to http post body +/** + * Provides a taint tracking configuration for reasoning about file data in outbound remote requests. */ import javascript -import semmle.javascript.frameworks.HTTP +import semmle.javascript.security.dataflow.RemoteFlowSources + +module FileAccessToHttp { -module FileAccessToHttpDataFlow { /** - * A data flow source for reasoning about file access to http post body flow vulnerabilities. + * A data flow source for file data in outbound remote requests. */ abstract class Source extends DataFlow::Node { } /** - * A data flow sink for reasoning about file access to http post body flow vulnerabilities. + * A data flow sink for file data in outbound remote requests. */ abstract class Sink extends DataFlow::Node { } /** - * A sanitizer for reasoning about file access to http post body flow vulnerabilities. + * A sanitizer for file data in outbound remote requests. */ abstract class Sanitizer extends DataFlow::Node { } /** - * A taint-tracking configuration for reasoning about file access to http post body flow vulnerabilities. + * A taint tracking configuration for file data in outbound remote requests. */ class Configuration extends TaintTracking::Configuration { - Configuration() { this = "FileAccessToHttpDataFlow" } + Configuration() { + this = "FileAccessToHttp" + } override predicate isSource(DataFlow::Node source) { source instanceof Source @@ -38,7 +41,7 @@ module FileAccessToHttpDataFlow { super.isSanitizer(node) or node instanceof Sanitizer } - + /** additional taint step that taints an object wrapping a source */ override predicate isAdditionalTaintStep(DataFlow::Node pred, DataFlow::Node succ) { ( @@ -53,7 +56,9 @@ module FileAccessToHttpDataFlow { } } - /** A source is a file access parameter, as in readFromFile(buffer). */ + /** + * A file access parameter, considered as a flow source for file data in outbound remote requests. + */ private class FileAccessArgumentAsSource extends Source { FileAccessArgumentAsSource() { exists(FileSystemReadAccess src | @@ -63,7 +68,7 @@ module FileAccessToHttpDataFlow { } /** - * The URL or data of a client request, viewed as a sink. + * The URL or data of a client request, considered as a flow source for file data in outbound remote requests. */ private class ClientRequestUrlOrDataAsSink extends Sink { ClientRequestUrlOrDataAsSink () { From 64b0d39390074829992170c4d48393183977fa3a Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Fri, 5 Oct 2018 15:32:52 +0200 Subject: [PATCH 45/58] JS: polish HttpToFileAccess.qll --- .../security/dataflow/HttpToFileAccess.qll | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/javascript/ql/src/semmle/javascript/security/dataflow/HttpToFileAccess.qll b/javascript/ql/src/semmle/javascript/security/dataflow/HttpToFileAccess.qll index a04f9c1e06a..10f3508d6d0 100644 --- a/javascript/ql/src/semmle/javascript/security/dataflow/HttpToFileAccess.qll +++ b/javascript/ql/src/semmle/javascript/security/dataflow/HttpToFileAccess.qll @@ -1,30 +1,33 @@ -/** - * Provides taint tracking configuration for reasoning about files created from untrusted http downloads. +/** + * Provides a taint tracking configuration for reasoning about user-controlled data in files. */ import javascript import semmle.javascript.security.dataflow.RemoteFlowSources -module HttpToFileAccessFlow { +module HttpToFileAccess { + /** - * A data flow source from untrusted http request to file access taint tracking configuration. + * A data flow source for user-controlled data in files. */ abstract class Source extends DataFlow::Node { } /** - * A data flow sink for untrusted http request to file access taint tracking configuration. + * A data flow sink for user-controlled data in files. */ abstract class Sink extends DataFlow::Node { } /** - * A sanitizer for untrusted http request to file access taint tracking configuration. + * A sanitizer for user-controlled data in files. */ abstract class Sanitizer extends DataFlow::Node { } /** - * A taint-tracking configuration for reasoning about file access from untrusted http response body. + * A taint tracking configuration for user-controlled data in files. */ class Configuration extends TaintTracking::Configuration { - Configuration() { this = "HttpToFileAccessFlow" } + Configuration() { + this = "HttpToFileAccess" + } override predicate isSource(DataFlow::Node source) { source instanceof Source @@ -39,12 +42,12 @@ module HttpToFileAccessFlow { node instanceof Sanitizer } } - - /** A source of remote data, considered as a flow source for untrusted http data to file system access. */ + + /** A source of remote user input, considered as a flow source for user-controlled data in files. */ class RemoteFlowSourceAsSource extends Source { RemoteFlowSourceAsSource() { this instanceof RemoteFlowSource } } - + /** A sink that represents file access method (write, append) argument */ class FileAccessAsSink extends Sink { FileAccessAsSink () { From 0da1ac4d751a8c92e619ffada9d180217ef96c69 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Mon, 8 Oct 2018 09:29:53 +0200 Subject: [PATCH 46/58] JS: naming and documentation cleanup for NodeJS file system accesses --- .../javascript/frameworks/NodeJSLib.qll | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll b/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll index 232e7e2564d..8525b0e6684 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll @@ -401,17 +401,19 @@ module NodeJSLib { ) } - /** * A call to a method from module `fs`, `graceful-fs` or `fs-extra`. */ - private class NodeJSFileSystemAccessCall extends FileSystemAccess, DataFlow::CallNode { + private class NodeJSFileSystemAccess extends FileSystemAccess, DataFlow::CallNode { string methodName; - NodeJSFileSystemAccessCall() { + NodeJSFileSystemAccess() { this = fsModuleMember(methodName).getACall() } + /** + * Gets the name of the called method. + */ string getMethodName() { result = methodName } @@ -422,9 +424,9 @@ module NodeJSLib { } } - /** Only NodeJSSystemFileAccessCalls that write data to 'fs' */ - private class NodeJSFileSystemAccessWriteCall extends FileSystemWriteAccess, NodeJSFileSystemAccessCall { - NodeJSFileSystemAccessWriteCall () { + /** A write to the file system. */ + private class NodeJSFileSystemAccessWrite extends FileSystemWriteAccess, NodeJSFileSystemAccess { + NodeJSFileSystemAccessWrite () { methodName = "appendFile" or methodName = "appendFileSync" or methodName = "write" or @@ -449,9 +451,9 @@ module NodeJSLib { } - /** Only NodeJSSystemFileAccessCalls that read data from 'fs' */ - private class NodeJSFileSystemAccessReadCall extends FileSystemReadAccess, NodeJSFileSystemAccessCall { - NodeJSFileSystemAccessReadCall () { + /** A file system read. */ + private class NodeJSFileSystemAccessRead extends FileSystemReadAccess, NodeJSFileSystemAccess { + NodeJSFileSystemAccessRead () { methodName = "read" or methodName = "readSync" or methodName = "readFile" or @@ -478,11 +480,11 @@ module NodeJSLib { } /** - * A write to the file system using a stream. + * A read from the file system. */ private class FileStreamWrite extends FileSystemWriteAccess, DataFlow::CallNode { - NodeJSFileSystemAccessCall stream; + NodeJSFileSystemAccess stream; FileStreamWrite() { stream.getMethodName() = "createWriteStream" and @@ -507,7 +509,7 @@ module NodeJSLib { */ private class FileStreamRead extends FileSystemReadAccess, DataFlow::CallNode { - NodeJSFileSystemAccessCall stream; + NodeJSFileSystemAccess stream; string method; From 6b8fd49fbacb4ff4c3178944447ef5fe531e8e9a Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Mon, 8 Oct 2018 13:48:55 +0200 Subject: [PATCH 47/58] JS: add change notes for two new queries --- change-notes/1.19/analysis-javascript.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/change-notes/1.19/analysis-javascript.md b/change-notes/1.19/analysis-javascript.md index bf36867b2eb..3321382ce50 100644 --- a/change-notes/1.19/analysis-javascript.md +++ b/change-notes/1.19/analysis-javascript.md @@ -16,10 +16,12 @@ | **Query** | **Tags** | **Purpose** | |-----------------------------------------------|------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | Enabling Node.js integration for Electron web content renderers (`js/enabling-electron-renderer-node-integration`) | security, frameworks/electron, external/cwe/cwe-094 | Highlights Electron web content renderer preferences with Node.js integration enabled, indicating a violation of [CWE-94](https://cwe.mitre.org/data/definitions/94.html). Results are not shown on LGTM by default. | +| File data in outbound remote request | security, external/cwe/cwe-200 | Highligts locations where file data is sent in a remote request. Results are not shown on LGTM by default. | | Host header poisoning in email generation | security, external/cwe/cwe-640 | Highlights code that generates emails with links that can be hijacked by HTTP host header poisoning, indicating a violation of [CWE-640](https://cwe.mitre.org/data/definitions/640.html). Results shown on LGTM by default. | | Replacement of a substring with itself (`js/identity-replacement`) | correctness, security, external/cwe/cwe-116 | Highlights string replacements that replace a string with itself, which usually indicates a mistake. Results shown on LGTM by default. | | Stored cross-site scripting (`js/stored-xss`) | security, external/cwe/cwe-079, external/cwe/cwe-116 | Highlights uncontrolled stored values flowing into HTML content, indicating a violation of [CWE-079](https://cwe.mitre.org/data/definitions/79.html). Results shown on LGTM by default. | | Unclear precedence of nested operators (`js/unclear-operator-precedence`) | maintainability, correctness, external/cwe/cwe-783 | Highlights nested binary operators whose relative precedence is easy to misunderstand. Results shown on LGTM by default. | +| User-controlled data in file | security, external/cwe/cwe-912 | Highlights locations where user-controlled data is written to a file. Results are not shown on LGTM by default. | ## Changes to existing queries From c885490c7e1cd8aedd4a7ea09a11d045261840ad Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Mon, 8 Oct 2018 13:53:44 +0200 Subject: [PATCH 48/58] JS: address review comments --- .../ql/src/semmle/javascript/frameworks/NodeJSLib.qll | 2 +- .../javascript/security/dataflow/FileAccessToHttp.qll | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll b/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll index 8525b0e6684..5f9ade38d2b 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll @@ -379,7 +379,7 @@ module NodeJSLib { * * We determine this by looking for an externs declaration for * `fs.methodName` where the `i`th parameter's name is `data` or - * `buffer` or a 'callback'. + * `buffer` or a `callback`. */ private predicate fsDataParam(string methodName, int i, string n) { exists (ExternalMemberDecl decl, Function f, JSDocParamTag p | diff --git a/javascript/ql/src/semmle/javascript/security/dataflow/FileAccessToHttp.qll b/javascript/ql/src/semmle/javascript/security/dataflow/FileAccessToHttp.qll index 42a92ce7ef9..0f5eda9afd0 100644 --- a/javascript/ql/src/semmle/javascript/security/dataflow/FileAccessToHttp.qll +++ b/javascript/ql/src/semmle/javascript/security/dataflow/FileAccessToHttp.qll @@ -42,13 +42,8 @@ module FileAccessToHttp { node instanceof Sanitizer } - /** additional taint step that taints an object wrapping a source */ override predicate isAdditionalTaintStep(DataFlow::Node pred, DataFlow::Node succ) { - ( - pred = DataFlow::valueNode(_) or - pred = DataFlow::parameterNode(_) or - pred instanceof DataFlow::PropRead - ) and + // taint entire object on property write exists (DataFlow::PropWrite pwr | succ = pwr.getBase() and pred = pwr.getRhs() From e93545d16eb1b2bead1e8a8382dd13bf7eaedd93 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Wed, 10 Oct 2018 15:28:42 +0200 Subject: [PATCH 49/58] JS: address more review comments --- change-notes/1.19/analysis-javascript.md | 2 +- .../ql/src/Security/CWE-912/HttpToFileAccess.ql | 2 +- .../semmle/javascript/frameworks/ClientRequests.qll | 2 +- .../src/semmle/javascript/frameworks/NodeJSLib.qll | 2 +- .../security/dataflow/HttpToFileAccess.qll | 12 ++++++------ 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/change-notes/1.19/analysis-javascript.md b/change-notes/1.19/analysis-javascript.md index 3321382ce50..bc476a98cf8 100644 --- a/change-notes/1.19/analysis-javascript.md +++ b/change-notes/1.19/analysis-javascript.md @@ -16,7 +16,7 @@ | **Query** | **Tags** | **Purpose** | |-----------------------------------------------|------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | Enabling Node.js integration for Electron web content renderers (`js/enabling-electron-renderer-node-integration`) | security, frameworks/electron, external/cwe/cwe-094 | Highlights Electron web content renderer preferences with Node.js integration enabled, indicating a violation of [CWE-94](https://cwe.mitre.org/data/definitions/94.html). Results are not shown on LGTM by default. | -| File data in outbound remote request | security, external/cwe/cwe-200 | Highligts locations where file data is sent in a remote request. Results are not shown on LGTM by default. | +| File data in outbound remote request | security, external/cwe/cwe-200 | Highlights locations where file data is sent in a remote request. Results are not shown on LGTM by default. | | Host header poisoning in email generation | security, external/cwe/cwe-640 | Highlights code that generates emails with links that can be hijacked by HTTP host header poisoning, indicating a violation of [CWE-640](https://cwe.mitre.org/data/definitions/640.html). Results shown on LGTM by default. | | Replacement of a substring with itself (`js/identity-replacement`) | correctness, security, external/cwe/cwe-116 | Highlights string replacements that replace a string with itself, which usually indicates a mistake. Results shown on LGTM by default. | | Stored cross-site scripting (`js/stored-xss`) | security, external/cwe/cwe-079, external/cwe/cwe-116 | Highlights uncontrolled stored values flowing into HTML content, indicating a violation of [CWE-079](https://cwe.mitre.org/data/definitions/79.html). Results shown on LGTM by default. | diff --git a/javascript/ql/src/Security/CWE-912/HttpToFileAccess.ql b/javascript/ql/src/Security/CWE-912/HttpToFileAccess.ql index 0b4344fe2e4..fa2aaf95e80 100644 --- a/javascript/ql/src/Security/CWE-912/HttpToFileAccess.ql +++ b/javascript/ql/src/Security/CWE-912/HttpToFileAccess.ql @@ -1,5 +1,5 @@ /** - * @name User-controlled data in file + * @name User-controlled data written to file * @description Writing user-controlled data directly to the file system allows arbitrary file upload and might indicate a backdoor. * @kind problem * @problem.severity warning diff --git a/javascript/ql/src/semmle/javascript/frameworks/ClientRequests.qll b/javascript/ql/src/semmle/javascript/frameworks/ClientRequests.qll index 470b533790b..41004c6123b 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/ClientRequests.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/ClientRequests.qll @@ -10,7 +10,7 @@ import javascript /** * A call that performs a request to a URL. * - * Example: An HTTP POST request is client request that sends some + * Example: An HTTP POST request is a client request that sends some * `data` to a `url`, where both the headers and the body of the request * contribute to the `data`. */ diff --git a/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll b/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll index 5f9ade38d2b..dd77a3b32c4 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/NodeJSLib.qll @@ -480,7 +480,7 @@ module NodeJSLib { } /** - * A read from the file system. + * A write to the file system, using a stream. */ private class FileStreamWrite extends FileSystemWriteAccess, DataFlow::CallNode { diff --git a/javascript/ql/src/semmle/javascript/security/dataflow/HttpToFileAccess.qll b/javascript/ql/src/semmle/javascript/security/dataflow/HttpToFileAccess.qll index 10f3508d6d0..8b47e4ed192 100644 --- a/javascript/ql/src/semmle/javascript/security/dataflow/HttpToFileAccess.qll +++ b/javascript/ql/src/semmle/javascript/security/dataflow/HttpToFileAccess.qll @@ -1,5 +1,5 @@ /** - * Provides a taint tracking configuration for reasoning about user-controlled data in files. + * Provides a taint tracking configuration for reasoning about writing user-controlled data to files. */ import javascript import semmle.javascript.security.dataflow.RemoteFlowSources @@ -7,22 +7,22 @@ import semmle.javascript.security.dataflow.RemoteFlowSources module HttpToFileAccess { /** - * A data flow source for user-controlled data in files. + * A data flow source for writing user-controlled data to files. */ abstract class Source extends DataFlow::Node { } /** - * A data flow sink for user-controlled data in files. + * A data flow sink for writing user-controlled data to files. */ abstract class Sink extends DataFlow::Node { } /** - * A sanitizer for user-controlled data in files. + * A sanitizer for writing user-controlled data to files. */ abstract class Sanitizer extends DataFlow::Node { } /** - * A taint tracking configuration for user-controlled data in files. + * A taint tracking configuration for writing user-controlled data to files. */ class Configuration extends TaintTracking::Configuration { Configuration() { @@ -43,7 +43,7 @@ module HttpToFileAccess { } } - /** A source of remote user input, considered as a flow source for user-controlled data in files. */ + /** A source of remote user input, considered as a flow source for writing user-controlled data to files. */ class RemoteFlowSourceAsSource extends Source { RemoteFlowSourceAsSource() { this instanceof RemoteFlowSource } } From 358b6c3413bb88e719f5e1d7b1a9c9df522a40e6 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Wed, 10 Oct 2018 15:32:57 +0200 Subject: [PATCH 50/58] JS: change "remote request" to "network request" --- change-notes/1.19/analysis-javascript.md | 2 +- .../ql/src/Security/CWE-200/FileAccessToHttp.ql | 6 +++--- .../ql/src/Security/CWE-918/RequestForgery.qhelp | 4 ++-- .../ql/src/Security/CWE-918/RequestForgery.ql | 4 ++-- .../security/dataflow/FileAccessToHttp.qll | 14 +++++++------- .../Security/CWE-200/FileAccessToHttp.expected | 16 ++++++++-------- 6 files changed, 23 insertions(+), 23 deletions(-) diff --git a/change-notes/1.19/analysis-javascript.md b/change-notes/1.19/analysis-javascript.md index bc476a98cf8..f00370ac34d 100644 --- a/change-notes/1.19/analysis-javascript.md +++ b/change-notes/1.19/analysis-javascript.md @@ -16,7 +16,7 @@ | **Query** | **Tags** | **Purpose** | |-----------------------------------------------|------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | Enabling Node.js integration for Electron web content renderers (`js/enabling-electron-renderer-node-integration`) | security, frameworks/electron, external/cwe/cwe-094 | Highlights Electron web content renderer preferences with Node.js integration enabled, indicating a violation of [CWE-94](https://cwe.mitre.org/data/definitions/94.html). Results are not shown on LGTM by default. | -| File data in outbound remote request | security, external/cwe/cwe-200 | Highlights locations where file data is sent in a remote request. Results are not shown on LGTM by default. | +| File data in outbound network request | security, external/cwe/cwe-200 | Highlights locations where file data is sent in a network request. Results are not shown on LGTM by default. | | Host header poisoning in email generation | security, external/cwe/cwe-640 | Highlights code that generates emails with links that can be hijacked by HTTP host header poisoning, indicating a violation of [CWE-640](https://cwe.mitre.org/data/definitions/640.html). Results shown on LGTM by default. | | Replacement of a substring with itself (`js/identity-replacement`) | correctness, security, external/cwe/cwe-116 | Highlights string replacements that replace a string with itself, which usually indicates a mistake. Results shown on LGTM by default. | | Stored cross-site scripting (`js/stored-xss`) | security, external/cwe/cwe-079, external/cwe/cwe-116 | Highlights uncontrolled stored values flowing into HTML content, indicating a violation of [CWE-079](https://cwe.mitre.org/data/definitions/79.html). Results shown on LGTM by default. | diff --git a/javascript/ql/src/Security/CWE-200/FileAccessToHttp.ql b/javascript/ql/src/Security/CWE-200/FileAccessToHttp.ql index cfa34da87fd..10b40d686a8 100644 --- a/javascript/ql/src/Security/CWE-200/FileAccessToHttp.ql +++ b/javascript/ql/src/Security/CWE-200/FileAccessToHttp.ql @@ -1,6 +1,6 @@ /** - * @name File data in outbound remote request - * @description Directly sending file data in an outbound remote request can indicate unauthorized information disclosure. + * @name File data in outbound network request + * @description Directly sending file data in an outbound network request can indicate unauthorized information disclosure. * @kind problem * @problem.severity warning * @id js/file-access-to-http @@ -13,4 +13,4 @@ import semmle.javascript.security.dataflow.FileAccessToHttp from FileAccessToHttp::Configuration config, DataFlow::Node src, DataFlow::Node sink where config.hasFlow (src, sink) -select sink, "$@ flows directly to outbound remote request", src, "File data" +select sink, "$@ flows directly to outbound network request", src, "File data" diff --git a/javascript/ql/src/Security/CWE-918/RequestForgery.qhelp b/javascript/ql/src/Security/CWE-918/RequestForgery.qhelp index 83b541d3a24..356edf65c86 100644 --- a/javascript/ql/src/Security/CWE-918/RequestForgery.qhelp +++ b/javascript/ql/src/Security/CWE-918/RequestForgery.qhelp @@ -25,8 +25,8 @@

To guard against request forgery, it is advisable to avoid - putting user input directly into a remote request. If a flexible - remote request mechanism is required, it is recommended to maintain a + putting user input directly into a network request. If a flexible + network request mechanism is required, it is recommended to maintain a list of authorized request targets and choose from that list based on the user input provided. diff --git a/javascript/ql/src/Security/CWE-918/RequestForgery.ql b/javascript/ql/src/Security/CWE-918/RequestForgery.ql index 8035f2ef56f..3c968bf9fa6 100644 --- a/javascript/ql/src/Security/CWE-918/RequestForgery.ql +++ b/javascript/ql/src/Security/CWE-918/RequestForgery.ql @@ -1,6 +1,6 @@ /** - * @name Uncontrolled data used in remote request - * @description Sending remote requests with user-controlled data allows for request forgery attacks. + * @name Uncontrolled data used in network request + * @description Sending network requests with user-controlled data allows for request forgery attacks. * @kind problem * @problem.severity error * @precision medium diff --git a/javascript/ql/src/semmle/javascript/security/dataflow/FileAccessToHttp.qll b/javascript/ql/src/semmle/javascript/security/dataflow/FileAccessToHttp.qll index 0f5eda9afd0..449ec25780f 100644 --- a/javascript/ql/src/semmle/javascript/security/dataflow/FileAccessToHttp.qll +++ b/javascript/ql/src/semmle/javascript/security/dataflow/FileAccessToHttp.qll @@ -1,5 +1,5 @@ /** - * Provides a taint tracking configuration for reasoning about file data in outbound remote requests. + * Provides a taint tracking configuration for reasoning about file data in outbound network requests. */ import javascript import semmle.javascript.security.dataflow.RemoteFlowSources @@ -7,22 +7,22 @@ import semmle.javascript.security.dataflow.RemoteFlowSources module FileAccessToHttp { /** - * A data flow source for file data in outbound remote requests. + * A data flow source for file data in outbound network requests. */ abstract class Source extends DataFlow::Node { } /** - * A data flow sink for file data in outbound remote requests. + * A data flow sink for file data in outbound network requests. */ abstract class Sink extends DataFlow::Node { } /** - * A sanitizer for file data in outbound remote requests. + * A sanitizer for file data in outbound network requests. */ abstract class Sanitizer extends DataFlow::Node { } /** - * A taint tracking configuration for file data in outbound remote requests. + * A taint tracking configuration for file data in outbound network requests. */ class Configuration extends TaintTracking::Configuration { Configuration() { @@ -52,7 +52,7 @@ module FileAccessToHttp { } /** - * A file access parameter, considered as a flow source for file data in outbound remote requests. + * A file access parameter, considered as a flow source for file data in outbound network requests. */ private class FileAccessArgumentAsSource extends Source { FileAccessArgumentAsSource() { @@ -63,7 +63,7 @@ module FileAccessToHttp { } /** - * The URL or data of a client request, considered as a flow source for file data in outbound remote requests. + * The URL or data of a client request, considered as a flow source for file data in outbound network requests. */ private class ClientRequestUrlOrDataAsSink extends Sink { ClientRequestUrlOrDataAsSink () { diff --git a/javascript/ql/test/query-tests/Security/CWE-200/FileAccessToHttp.expected b/javascript/ql/test/query-tests/Security/CWE-200/FileAccessToHttp.expected index 936c9e0b488..34dbdec5381 100644 --- a/javascript/ql/test/query-tests/Security/CWE-200/FileAccessToHttp.expected +++ b/javascript/ql/test/query-tests/Security/CWE-200/FileAccessToHttp.expected @@ -1,8 +1,8 @@ -| bufferRead.js:33:21:33:28 | postData | $@ flows directly to outbound remote request | bufferRead.js:12:22:12:43 | new Buf ... s.size) | File data | -| googlecompiler.js:38:18:38:26 | post_data | $@ flows directly to outbound remote request | googlecompiler.js:44:54:44:57 | data | File data | -| readFileSync.js:26:18:26:18 | s | $@ flows directly to outbound remote request | readFileSync.js:5:12:5:39 | fs.read ... t.txt") | File data | -| readStreamRead.js:30:19:30:23 | chunk | $@ flows directly to outbound remote request | readStreamRead.js:13:21:13:35 | readable.read() | File data | -| request.js:8:11:8:20 | {jsonData} | $@ flows directly to outbound remote request | request.js:28:52:28:55 | data | File data | -| request.js:16:11:23:3 | {\\n u ... ody\\n } | $@ flows directly to outbound remote request | request.js:43:51:43:54 | data | File data | -| sentAsHeaders.js:14:20:19:9 | {\\n ... } | $@ flows directly to outbound remote request | sentAsHeaders.js:10:79:10:84 | buffer | File data | -| sentAsHeaders.js:20:20:25:9 | {\\n ... } | $@ flows directly to outbound remote request | sentAsHeaders.js:10:79:10:84 | buffer | File data | +| bufferRead.js:33:21:33:28 | postData | $@ flows directly to outbound network request | bufferRead.js:12:22:12:43 | new Buf ... s.size) | File data | +| googlecompiler.js:38:18:38:26 | post_data | $@ flows directly to outbound network request | googlecompiler.js:44:54:44:57 | data | File data | +| readFileSync.js:26:18:26:18 | s | $@ flows directly to outbound network request | readFileSync.js:5:12:5:39 | fs.read ... t.txt") | File data | +| readStreamRead.js:30:19:30:23 | chunk | $@ flows directly to outbound network request | readStreamRead.js:13:21:13:35 | readable.read() | File data | +| request.js:8:11:8:20 | {jsonData} | $@ flows directly to outbound network request | request.js:28:52:28:55 | data | File data | +| request.js:16:11:23:3 | {\\n u ... ody\\n } | $@ flows directly to outbound network request | request.js:43:51:43:54 | data | File data | +| sentAsHeaders.js:14:20:19:9 | {\\n ... } | $@ flows directly to outbound network request | sentAsHeaders.js:10:79:10:84 | buffer | File data | +| sentAsHeaders.js:20:20:25:9 | {\\n ... } | $@ flows directly to outbound network request | sentAsHeaders.js:10:79:10:84 | buffer | File data | From 103d140e712fade21f80742d744380195cdf8eda Mon Sep 17 00:00:00 2001 From: calum Date: Mon, 8 Oct 2018 12:47:37 +0100 Subject: [PATCH 51/58] C#: Migrate extractor to this repository. --- csharp/extractor/.gitignore | 16 + csharp/extractor/CSharpExtractor.sln | 83 ++ .../Semmle.Autobuild.Tests/BuildScripts.cs | 883 +++++++++++ .../Properties/AssemblyInfo.cs | 32 + .../Semmle.Autobuild.Tests.csproj | 18 + .../Semmle.Autobuild/AspBuildRule.cs | 20 + .../Semmle.Autobuild/AutobuildOptions.cs | 103 ++ .../extractor/Semmle.Autobuild/Autobuilder.cs | 352 +++++ .../Semmle.Autobuild/BuildActions.cs | 176 +++ .../Semmle.Autobuild/BuildCommandAutoRule.cs | 61 + .../Semmle.Autobuild/BuildCommandRule.cs | 28 + .../extractor/Semmle.Autobuild/BuildScript.cs | 274 ++++ .../extractor/Semmle.Autobuild/BuildTools.cs | 99 ++ .../Semmle.Autobuild/CommandBuilder.cs | 195 +++ .../extractor/Semmle.Autobuild/DotNetRule.cs | 268 ++++ csharp/extractor/Semmle.Autobuild/Language.cs | 24 + .../extractor/Semmle.Autobuild/MsBuildRule.cs | 118 ++ csharp/extractor/Semmle.Autobuild/Program.cs | 28 + csharp/extractor/Semmle.Autobuild/Project.cs | 69 + .../Properties/AssemblyInfo.cs | 35 + .../Semmle.Autobuild/Semmle.Autobuild.csproj | 27 + csharp/extractor/Semmle.Autobuild/Solution.cs | 107 ++ .../Semmle.Autobuild/StandaloneBuildRule.cs | 46 + .../Semmle.Autobuild/XmlBuildRule.cs | 18 + .../ExtractorOptions.cs | 267 ++++ .../Semmle.Extraction.CIL.Driver/Program.cs | 68 + .../Properties/AssemblyInfo.cs | 35 + .../Semmle.Extraction.CIL.Driver.csproj | 21 + .../Semmle.Extraction.CIL/CachedFunction.cs | 69 + .../Semmle.Extraction.CIL/Context.cs | 173 +++ .../Entities/Assembly.cs | 152 ++ .../Entities/Attribute.cs | 100 ++ .../Semmle.Extraction.CIL/Entities/Event.cs | 67 + .../Entities/ExceptionRegion.cs | 63 + .../Semmle.Extraction.CIL/Entities/Field.cs | 147 ++ .../Semmle.Extraction.CIL/Entities/File.cs | 66 + .../Semmle.Extraction.CIL/Entities/Folder.cs | 42 + .../Entities/Instruction.cs | 486 +++++++ .../Entities/LocalVariable.cs | 36 + .../Semmle.Extraction.CIL/Entities/Method.cs | 515 +++++++ .../Entities/Namespace.cs | 78 + .../Entities/Parameter.cs | 43 + .../Entities/Property.cs | 64 + .../Entities/SourceLocation.cs | 37 + .../Semmle.Extraction.CIL/Entities/Type.cs | 1285 +++++++++++++++++ .../ExtractionProduct.cs | 127 ++ .../Semmle.Extraction.CIL/Factories.cs | 233 +++ csharp/extractor/Semmle.Extraction.CIL/Id.cs | 204 +++ .../PDB/MetadataPdbReader.cs | 93 ++ .../PDB/NativePdbReader.cs | 143 ++ .../Semmle.Extraction.CIL/PDB/PdbReader.cs | 151 ++ .../Properties/AssemblyInfo.cs | 35 + .../Semmle.Extraction.CIL.csproj | 26 + .../extractor/Semmle.Extraction.CIL/Tuples.cs | 209 +++ .../Semmle.Extraction.CSharp.Driver/Driver.cs | 13 + .../Properties/AssemblyInfo.cs | 35 + .../Semmle.Extraction.CSharp.Driver.csproj | 15 + .../AssemblyCache.cs | 184 +++ .../AssemblyInfo.cs | 179 +++ .../BuildAnalysis.cs | 312 ++++ .../CsProjFile.cs | 120 ++ .../NugetPackages.cs | 195 +++ .../Options.cs | 178 +++ .../Program.cs | 158 ++ .../ProgressMonitor.cs | 105 ++ .../Properties/AssemblyInfo.cs | 35 + .../Runtime.cs | 74 + ...Semmle.Extraction.CSharp.Standalone.csproj | 30 + .../SolutionFile.cs | 68 + .../Semmle.Extraction.CSharp/Analyser.cs | 505 +++++++ .../CompilerVersion.cs | 107 ++ .../Entities/Accessor.cs | 88 ++ .../Entities/Attribute.cs | 79 + .../Entities/CommentBlock.cs | 51 + .../Entities/CommentLine.cs | 147 ++ .../Entities/Constructor.cs | 158 ++ .../Entities/Conversion.cs | 37 + .../Entities/Destructor.cs | 35 + .../Entities/Event.cs | 76 + .../Entities/EventAccessor.cs | 64 + .../Entities/Expression.cs | 527 +++++++ .../Entities/Expressions/Access.cs | 55 + .../Entities/Expressions/ArgList.cs | 18 + .../Entities/Expressions/ArrayCreation.cs | 108 ++ .../Entities/Expressions/Assignment.cs | 154 ++ .../Entities/Expressions/Await.cs | 17 + .../Entities/Expressions/Base.cs | 12 + .../Entities/Expressions/Binary.cs | 61 + .../Entities/Expressions/Cast.cs | 29 + .../Entities/Expressions/Checked.cs | 17 + .../Entities/Expressions/Conditional.cs | 19 + .../Entities/Expressions/Default.cs | 17 + .../Entities/Expressions/Discard.cs | 16 + .../Entities/Expressions/ElementAccess.cs | 95 ++ .../Entities/Expressions/Factory.cs | 242 ++++ .../Entities/Expressions/ImplicitCast.cs | 92 ++ .../Entities/Expressions/Initializer.cs | 137 ++ .../Expressions/InterpolatedString.cs | 36 + .../Entities/Expressions/Invocation.cs | 181 +++ .../Entities/Expressions/IsPattern.cs | 46 + .../Entities/Expressions/Lambda.cs | 56 + .../Entities/Expressions/Literal.cs | 72 + .../Entities/Expressions/MakeRef.cs | 17 + .../Entities/Expressions/MemberAccess.cs | 101 ++ .../Entities/Expressions/Name.cs | 66 + .../Entities/Expressions/ObjectCreation.cs | 136 ++ .../Expressions/PointerMemberAccess.cs | 19 + .../Entities/Expressions/PostfixUnary.cs | 32 + .../Entities/Expressions/Query.cs | 260 ++++ .../Entities/Expressions/Ref.cs | 17 + .../Entities/Expressions/RefType.cs | 17 + .../Entities/Expressions/RefValue.cs | 18 + .../Entities/Expressions/Sizeof.cs | 17 + .../Entities/Expressions/This.cs | 16 + .../Entities/Expressions/Throw.cs | 17 + .../Entities/Expressions/Tuple.cs | 24 + .../Entities/Expressions/TypeAccess.cs | 38 + .../Entities/Expressions/TypeOf.cs | 17 + .../Entities/Expressions/Unary.cs | 35 + .../Entities/Expressions/Unchecked.cs | 17 + .../Entities/Expressions/Unknown.cs | 10 + .../Expressions/VariableDeclaration.cs | 145 ++ .../Entities/Field.cs | 101 ++ .../Entities/Indexer.cs | 138 ++ .../Entities/LocalFunction.cs | 48 + .../Entities/LocalVariable.cs | 95 ++ .../Entities/Method.cs | 368 +++++ .../Entities/Modifier.cs | 169 +++ .../Entities/Namespace.cs | 45 + .../Entities/NamespaceDeclaration.cs | 41 + .../Entities/OrdinaryMethod.cs | 60 + .../Entities/Parameter.cs | 290 ++++ .../Entities/Property.cs | 119 ++ .../Entities/Statement.cs | 79 + .../Entities/Statements/Block.cs | 28 + .../Entities/Statements/Break.cs | 20 + .../Entities/Statements/Case.cs | 105 ++ .../Entities/Statements/Catch.cs | 51 + .../Entities/Statements/Checked.cs | 23 + .../Entities/Statements/Continue.cs | 21 + .../Entities/Statements/Do.cs | 25 + .../Entities/Statements/Empty.cs | 20 + .../Statements/ExpressionStatement.cs | 25 + .../Entities/Statements/Factory.cs | 73 + .../Entities/Statements/Fixed.cs | 25 + .../Entities/Statements/For.cs | 45 + .../Entities/Statements/ForEach.cs | 56 + .../Entities/Statements/Goto.cs | 57 + .../Entities/Statements/If.cs | 28 + .../Entities/Statements/Labeled.cs | 38 + .../Entities/Statements/LocalDeclaration.cs | 25 + .../Entities/Statements/LocalFunction.cs | 48 + .../Entities/Statements/Lock.cs | 24 + .../Entities/Statements/Return.cs | 24 + .../Entities/Statements/Switch.cs | 49 + .../Entities/Statements/Throw.cs | 24 + .../Entities/Statements/Try.cs | 47 + .../Entities/Statements/Unchecked.cs | 23 + .../Entities/Statements/Unsafe.cs | 23 + .../Entities/Statements/Using.cs | 31 + .../Entities/Statements/While.cs | 24 + .../Entities/Statements/Yield.cs | 26 + .../Entities/Symbol.cs | 102 ++ .../Entities/TypeMention.cs | 93 ++ .../Entities/Types/ArrayType.cs | 53 + .../Entities/Types/DynamicType.cs | 35 + .../Entities/Types/NamedType.cs | 204 +++ .../Entities/Types/NullType.cs | 35 + .../Entities/Types/PointerType.cs | 36 + .../Entities/Types/TupleType.cs | 68 + .../Entities/Types/Type.cs | 327 +++++ .../Entities/Types/TypeParameter.cs | 131 ++ .../Types/TypeParameterConstraints.cs | 10 + .../Entities/UserOperator.cs | 207 +++ .../Entities/UsingDirective.cs | 61 + .../Semmle.Extraction.CSharp/Extractor.cs | 407 ++++++ .../Kinds/ExprKind.cs | 110 ++ .../Kinds/StmtKind.cs | 39 + .../Kinds/TypeKind.cs | 39 + .../Semmle.Extraction.CSharp/Options.cs | 87 ++ .../Populators/Ast.cs | 69 + .../Populators/Comments.cs | 33 + .../Populators/CompilationUnit.cs | 125 ++ .../Populators/Locations.cs | 104 ++ .../Populators/Methods.cs | 64 + .../Populators/Symbols.cs | 136 ++ .../Properties/AssemblyInfo.cs | 35 + .../Semmle.Extraction.CSharp.csproj | 25 + .../SymbolExtensions.cs | 371 +++++ .../Semmle.Extraction.CSharp/Tuples.cs | 215 +++ .../Semmle.Extraction.Tests/Layout.cs | 200 +++ .../Semmle.Extraction.Tests/Options.cs | 188 +++ .../Properties/AssemblyInfo.cs | 35 + .../Semmle.Extraction.Tests.csproj | 21 + .../Semmle.Extraction.Tests/TrapWriter.cs | 67 + .../Semmle.Extraction/CommentProcessing.cs | 482 +++++++ csharp/extractor/Semmle.Extraction/Context.cs | 496 +++++++ .../Semmle.Extraction/Entities/Assembly.cs | 77 + .../Semmle.Extraction/Entities/File.cs | 124 ++ .../Semmle.Extraction/Entities/Folder.cs | 55 + .../Entities/GeneratedLocation.cs | 33 + .../Semmle.Extraction/Entities/Location.cs | 30 + .../Entities/SourceLocation.cs | 52 + csharp/extractor/Semmle.Extraction/Entity.cs | 143 ++ .../Semmle.Extraction/ExtractionScope.cs | 63 + .../extractor/Semmle.Extraction/Extractor.cs | 208 +++ .../Semmle.Extraction/FreshEntity.cs | 31 + csharp/extractor/Semmle.Extraction/Id.cs | 193 +++ .../Semmle.Extraction/InternalError.cs | 34 + csharp/extractor/Semmle.Extraction/Layout.cs | 242 ++++ csharp/extractor/Semmle.Extraction/Message.cs | 18 + csharp/extractor/Semmle.Extraction/Options.cs | 95 ++ .../Properties/AssemblyInfo.cs | 35 + .../Semmle.Extraction.csproj | 22 + csharp/extractor/Semmle.Extraction/Symbol.cs | 63 + .../Semmle.Extraction/TrapBuilder.cs | 96 ++ .../extractor/Semmle.Extraction/TrapWriter.cs | 306 ++++ csharp/extractor/Semmle.Extraction/Tuple.cs | 87 ++ csharp/extractor/Semmle.Extraction/Tuples.cs | 32 + .../extractor/Semmle.Util.Tests/ActionMap.cs | 55 + .../Semmle.Util.Tests/CanonicalPathCache.cs | 192 +++ .../extractor/Semmle.Util.Tests/FileUtils.cs | 20 + .../Semmle.Util.Tests/LineCounterTest.cs | 81 ++ .../extractor/Semmle.Util.Tests/LongPaths.cs | 184 +++ .../Properties/AssemblyInfo.cs | 35 + .../Semmle.Util.Tests.csproj | 18 + .../extractor/Semmle.Util.Tests/TextTest.cs | 78 + csharp/extractor/Semmle.Util/ActionMap.cs | 47 + .../Semmle.Util/CanonicalPathCache.cs | 326 +++++ .../Semmle.Util/CommandLineOptions.cs | 75 + .../Semmle.Util/DictionaryExtensions.cs | 23 + csharp/extractor/Semmle.Util/Enumerators.cs | 19 + csharp/extractor/Semmle.Util/FileUtils.cs | 54 + .../extractor/Semmle.Util/FuzzyDictionary.cs | 165 +++ .../Semmle.Util/IEnumerableExtensions.cs | 90 ++ csharp/extractor/Semmle.Util/LineCounter.cs | 276 ++++ csharp/extractor/Semmle.Util/Logger.cs | 197 +++ csharp/extractor/Semmle.Util/LoggerUtils.cs | 39 + .../Semmle.Util/ProcessStartInfoExtensions.cs | 29 + .../Semmle.Util/Properties/AssemblyInfo.cs | 35 + .../extractor/Semmle.Util/Semmle.Util.csproj | 18 + .../extractor/Semmle.Util/SharedReference.cs | 18 + csharp/extractor/Semmle.Util/StringBuilder.cs | 43 + csharp/extractor/Semmle.Util/Text.cs | 109 ++ csharp/extractor/Semmle.Util/Win32.cs | 78 + csharp/extractor/Semmle.Util/Worklist.cs | 58 + csharp/extractor/global.json | 5 + 247 files changed, 25866 insertions(+) create mode 100644 csharp/extractor/.gitignore create mode 100644 csharp/extractor/CSharpExtractor.sln create mode 100644 csharp/extractor/Semmle.Autobuild.Tests/BuildScripts.cs create mode 100644 csharp/extractor/Semmle.Autobuild.Tests/Properties/AssemblyInfo.cs create mode 100644 csharp/extractor/Semmle.Autobuild.Tests/Semmle.Autobuild.Tests.csproj create mode 100644 csharp/extractor/Semmle.Autobuild/AspBuildRule.cs create mode 100644 csharp/extractor/Semmle.Autobuild/AutobuildOptions.cs create mode 100644 csharp/extractor/Semmle.Autobuild/Autobuilder.cs create mode 100644 csharp/extractor/Semmle.Autobuild/BuildActions.cs create mode 100644 csharp/extractor/Semmle.Autobuild/BuildCommandAutoRule.cs create mode 100644 csharp/extractor/Semmle.Autobuild/BuildCommandRule.cs create mode 100644 csharp/extractor/Semmle.Autobuild/BuildScript.cs create mode 100644 csharp/extractor/Semmle.Autobuild/BuildTools.cs create mode 100644 csharp/extractor/Semmle.Autobuild/CommandBuilder.cs create mode 100644 csharp/extractor/Semmle.Autobuild/DotNetRule.cs create mode 100644 csharp/extractor/Semmle.Autobuild/Language.cs create mode 100644 csharp/extractor/Semmle.Autobuild/MsBuildRule.cs create mode 100644 csharp/extractor/Semmle.Autobuild/Program.cs create mode 100644 csharp/extractor/Semmle.Autobuild/Project.cs create mode 100644 csharp/extractor/Semmle.Autobuild/Properties/AssemblyInfo.cs create mode 100644 csharp/extractor/Semmle.Autobuild/Semmle.Autobuild.csproj create mode 100644 csharp/extractor/Semmle.Autobuild/Solution.cs create mode 100644 csharp/extractor/Semmle.Autobuild/StandaloneBuildRule.cs create mode 100644 csharp/extractor/Semmle.Autobuild/XmlBuildRule.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL.Driver/ExtractorOptions.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL.Driver/Properties/AssemblyInfo.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL.Driver/Semmle.Extraction.CIL.Driver.csproj create mode 100644 csharp/extractor/Semmle.Extraction.CIL/CachedFunction.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/Context.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/Entities/Attribute.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/Entities/Event.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/Entities/ExceptionRegion.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/Entities/Field.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/Entities/Instruction.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/Entities/LocalVariable.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/Entities/Method.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/Entities/Parameter.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/Entities/Property.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/ExtractionProduct.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/Factories.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/Id.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/PDB/MetadataPdbReader.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/PDB/NativePdbReader.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/PDB/PdbReader.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/Properties/AssemblyInfo.cs create mode 100644 csharp/extractor/Semmle.Extraction.CIL/Semmle.Extraction.CIL.csproj create mode 100644 csharp/extractor/Semmle.Extraction.CIL/Tuples.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.Driver/Driver.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/AssemblyInfo.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.Driver/Semmle.Extraction.CSharp.Driver.csproj create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyInfo.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.Standalone/Options.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.Standalone/Properties/AssemblyInfo.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.Standalone/Runtime.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.Standalone/SolutionFile.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/CompilerVersion.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Accessor.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentBlock.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Constructor.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Conversion.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Destructor.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Event.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/EventAccessor.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expression.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Access.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ArgList.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ArrayCreation.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Assignment.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Await.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Base.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Binary.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Cast.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Checked.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Conditional.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Default.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Discard.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ElementAccess.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Factory.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ImplicitCast.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Initializer.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/InterpolatedString.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Invocation.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/IsPattern.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Lambda.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Literal.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MakeRef.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MemberAccess.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Name.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ObjectCreation.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/PointerMemberAccess.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/PostfixUnary.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Query.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Ref.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/RefType.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/RefValue.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Sizeof.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/This.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Throw.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Tuple.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/TypeAccess.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/TypeOf.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Unary.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Unchecked.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Unknown.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/VariableDeclaration.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Indexer.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalFunction.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalVariable.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Method.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Namespace.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statement.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Block.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Break.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Case.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Catch.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Checked.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Continue.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Do.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Empty.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/ExpressionStatement.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Factory.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Fixed.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/For.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/ForEach.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Goto.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/If.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Labeled.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/LocalDeclaration.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/LocalFunction.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Lock.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Return.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Switch.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Throw.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Try.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Unchecked.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Unsafe.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Using.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/While.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Yield.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Symbol.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/TypeMention.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/ArrayType.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/DynamicType.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NullType.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/PointerType.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TupleType.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameter.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameterConstraints.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/UserOperator.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Extractor.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Kinds/ExprKind.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Kinds/StmtKind.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Kinds/TypeKind.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Options.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Populators/Ast.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Populators/Comments.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Populators/CompilationUnit.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Populators/Locations.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Populators/Methods.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Populators/Symbols.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Properties/AssemblyInfo.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Semmle.Extraction.CSharp.csproj create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/SymbolExtensions.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Tuples.cs create mode 100644 csharp/extractor/Semmle.Extraction.Tests/Layout.cs create mode 100644 csharp/extractor/Semmle.Extraction.Tests/Options.cs create mode 100644 csharp/extractor/Semmle.Extraction.Tests/Properties/AssemblyInfo.cs create mode 100644 csharp/extractor/Semmle.Extraction.Tests/Semmle.Extraction.Tests.csproj create mode 100644 csharp/extractor/Semmle.Extraction.Tests/TrapWriter.cs create mode 100644 csharp/extractor/Semmle.Extraction/CommentProcessing.cs create mode 100644 csharp/extractor/Semmle.Extraction/Context.cs create mode 100644 csharp/extractor/Semmle.Extraction/Entities/Assembly.cs create mode 100644 csharp/extractor/Semmle.Extraction/Entities/File.cs create mode 100644 csharp/extractor/Semmle.Extraction/Entities/Folder.cs create mode 100644 csharp/extractor/Semmle.Extraction/Entities/GeneratedLocation.cs create mode 100644 csharp/extractor/Semmle.Extraction/Entities/Location.cs create mode 100644 csharp/extractor/Semmle.Extraction/Entities/SourceLocation.cs create mode 100644 csharp/extractor/Semmle.Extraction/Entity.cs create mode 100644 csharp/extractor/Semmle.Extraction/ExtractionScope.cs create mode 100644 csharp/extractor/Semmle.Extraction/Extractor.cs create mode 100644 csharp/extractor/Semmle.Extraction/FreshEntity.cs create mode 100644 csharp/extractor/Semmle.Extraction/Id.cs create mode 100644 csharp/extractor/Semmle.Extraction/InternalError.cs create mode 100644 csharp/extractor/Semmle.Extraction/Layout.cs create mode 100644 csharp/extractor/Semmle.Extraction/Message.cs create mode 100644 csharp/extractor/Semmle.Extraction/Options.cs create mode 100644 csharp/extractor/Semmle.Extraction/Properties/AssemblyInfo.cs create mode 100644 csharp/extractor/Semmle.Extraction/Semmle.Extraction.csproj create mode 100644 csharp/extractor/Semmle.Extraction/Symbol.cs create mode 100644 csharp/extractor/Semmle.Extraction/TrapBuilder.cs create mode 100644 csharp/extractor/Semmle.Extraction/TrapWriter.cs create mode 100644 csharp/extractor/Semmle.Extraction/Tuple.cs create mode 100644 csharp/extractor/Semmle.Extraction/Tuples.cs create mode 100644 csharp/extractor/Semmle.Util.Tests/ActionMap.cs create mode 100644 csharp/extractor/Semmle.Util.Tests/CanonicalPathCache.cs create mode 100644 csharp/extractor/Semmle.Util.Tests/FileUtils.cs create mode 100644 csharp/extractor/Semmle.Util.Tests/LineCounterTest.cs create mode 100644 csharp/extractor/Semmle.Util.Tests/LongPaths.cs create mode 100644 csharp/extractor/Semmle.Util.Tests/Properties/AssemblyInfo.cs create mode 100644 csharp/extractor/Semmle.Util.Tests/Semmle.Util.Tests.csproj create mode 100644 csharp/extractor/Semmle.Util.Tests/TextTest.cs create mode 100644 csharp/extractor/Semmle.Util/ActionMap.cs create mode 100644 csharp/extractor/Semmle.Util/CanonicalPathCache.cs create mode 100644 csharp/extractor/Semmle.Util/CommandLineOptions.cs create mode 100644 csharp/extractor/Semmle.Util/DictionaryExtensions.cs create mode 100644 csharp/extractor/Semmle.Util/Enumerators.cs create mode 100644 csharp/extractor/Semmle.Util/FileUtils.cs create mode 100644 csharp/extractor/Semmle.Util/FuzzyDictionary.cs create mode 100644 csharp/extractor/Semmle.Util/IEnumerableExtensions.cs create mode 100644 csharp/extractor/Semmle.Util/LineCounter.cs create mode 100644 csharp/extractor/Semmle.Util/Logger.cs create mode 100644 csharp/extractor/Semmle.Util/LoggerUtils.cs create mode 100644 csharp/extractor/Semmle.Util/ProcessStartInfoExtensions.cs create mode 100644 csharp/extractor/Semmle.Util/Properties/AssemblyInfo.cs create mode 100644 csharp/extractor/Semmle.Util/Semmle.Util.csproj create mode 100644 csharp/extractor/Semmle.Util/SharedReference.cs create mode 100644 csharp/extractor/Semmle.Util/StringBuilder.cs create mode 100644 csharp/extractor/Semmle.Util/Text.cs create mode 100644 csharp/extractor/Semmle.Util/Win32.cs create mode 100644 csharp/extractor/Semmle.Util/Worklist.cs create mode 100644 csharp/extractor/global.json diff --git a/csharp/extractor/.gitignore b/csharp/extractor/.gitignore new file mode 100644 index 00000000000..786c13ed7b3 --- /dev/null +++ b/csharp/extractor/.gitignore @@ -0,0 +1,16 @@ +obj/ +SemmleTests/bin +TestResults/ +*.manifest +*.pdb +*.suo +*.mdb +*.vsmdi +CSharpExtractor*.config +csharp.log +**/bin/Debug +**/bin/Release +*.tlog +.vs +packages +*.user \ No newline at end of file diff --git a/csharp/extractor/CSharpExtractor.sln b/csharp/extractor/CSharpExtractor.sln new file mode 100644 index 00000000000..9f0d615ed38 --- /dev/null +++ b/csharp/extractor/CSharpExtractor.sln @@ -0,0 +1,83 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.27130.2036 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Util", "Semmle.Util\Semmle.Util.csproj", "{CDD7AD69-0FD8-40F0-A9DA-F1077A2A85D6}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction", "Semmle.Extraction\Semmle.Extraction.csproj", "{81EAAD75-4BE1-44E4-91DF-20778216DB64}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp", "Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj", "{C4D62DA0-B64B-440B-86DC-AB52318CB8BF}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CIL", "Semmle.Extraction.CIL\Semmle.Extraction.CIL.csproj", "{399A1579-68F0-40F4-9A23-F241BA697F9C}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Autobuild", "Semmle.Autobuild\Semmle.Autobuild.csproj", "{5131EF00-0BA9-4436-A3B0-C5CDAB4B194C}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.Standalone", "Semmle.Extraction.CSharp.Standalone\Semmle.Extraction.CSharp.Standalone.csproj", "{D00E7D25-0FA0-48EC-B048-CD60CE1B30D8}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CIL.Driver", "Semmle.Extraction.CIL.Driver\Semmle.Extraction.CIL.Driver.csproj", "{EFA400B3-C1CE-446F-A4E2-8B44E61EF47C}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.Driver", "Semmle.Extraction.CSharp.Driver\Semmle.Extraction.CSharp.Driver.csproj", "{C36453BF-0C82-448A-B15D-26947503A2D3}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.Tests", "Semmle.Extraction.Tests\Semmle.Extraction.Tests.csproj", "{CD8D3F90-AD2E-4BB5-8E82-B94AA293864A}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Util.Tests", "Semmle.Util.Tests\Semmle.Util.Tests.csproj", "{55A620F0-23F6-440D-A5BA-0567613B3C0F}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Autobuild.Tests", "Semmle.Autobuild.Tests\Semmle.Autobuild.Tests.csproj", "{CE267461-D762-4F53-A275-685A0A4EC48D}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {CDD7AD69-0FD8-40F0-A9DA-F1077A2A85D6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CDD7AD69-0FD8-40F0-A9DA-F1077A2A85D6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CDD7AD69-0FD8-40F0-A9DA-F1077A2A85D6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CDD7AD69-0FD8-40F0-A9DA-F1077A2A85D6}.Release|Any CPU.Build.0 = Release|Any CPU + {81EAAD75-4BE1-44E4-91DF-20778216DB64}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {81EAAD75-4BE1-44E4-91DF-20778216DB64}.Debug|Any CPU.Build.0 = Debug|Any CPU + {81EAAD75-4BE1-44E4-91DF-20778216DB64}.Release|Any CPU.ActiveCfg = Release|Any CPU + {81EAAD75-4BE1-44E4-91DF-20778216DB64}.Release|Any CPU.Build.0 = Release|Any CPU + {C4D62DA0-B64B-440B-86DC-AB52318CB8BF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C4D62DA0-B64B-440B-86DC-AB52318CB8BF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C4D62DA0-B64B-440B-86DC-AB52318CB8BF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C4D62DA0-B64B-440B-86DC-AB52318CB8BF}.Release|Any CPU.Build.0 = Release|Any CPU + {399A1579-68F0-40F4-9A23-F241BA697F9C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {399A1579-68F0-40F4-9A23-F241BA697F9C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {399A1579-68F0-40F4-9A23-F241BA697F9C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {399A1579-68F0-40F4-9A23-F241BA697F9C}.Release|Any CPU.Build.0 = Release|Any CPU + {5131EF00-0BA9-4436-A3B0-C5CDAB4B194C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5131EF00-0BA9-4436-A3B0-C5CDAB4B194C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5131EF00-0BA9-4436-A3B0-C5CDAB4B194C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5131EF00-0BA9-4436-A3B0-C5CDAB4B194C}.Release|Any CPU.Build.0 = Release|Any CPU + {D00E7D25-0FA0-48EC-B048-CD60CE1B30D8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D00E7D25-0FA0-48EC-B048-CD60CE1B30D8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D00E7D25-0FA0-48EC-B048-CD60CE1B30D8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D00E7D25-0FA0-48EC-B048-CD60CE1B30D8}.Release|Any CPU.Build.0 = Release|Any CPU + {EFA400B3-C1CE-446F-A4E2-8B44E61EF47C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EFA400B3-C1CE-446F-A4E2-8B44E61EF47C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EFA400B3-C1CE-446F-A4E2-8B44E61EF47C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EFA400B3-C1CE-446F-A4E2-8B44E61EF47C}.Release|Any CPU.Build.0 = Release|Any CPU + {C36453BF-0C82-448A-B15D-26947503A2D3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C36453BF-0C82-448A-B15D-26947503A2D3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C36453BF-0C82-448A-B15D-26947503A2D3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C36453BF-0C82-448A-B15D-26947503A2D3}.Release|Any CPU.Build.0 = Release|Any CPU + {CD8D3F90-AD2E-4BB5-8E82-B94AA293864A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CD8D3F90-AD2E-4BB5-8E82-B94AA293864A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CD8D3F90-AD2E-4BB5-8E82-B94AA293864A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {55A620F0-23F6-440D-A5BA-0567613B3C0F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {55A620F0-23F6-440D-A5BA-0567613B3C0F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {55A620F0-23F6-440D-A5BA-0567613B3C0F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CE267461-D762-4F53-A275-685A0A4EC48D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CE267461-D762-4F53-A275-685A0A4EC48D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CE267461-D762-4F53-A275-685A0A4EC48D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CE267461-D762-4F53-A275-685A0A4EC48D}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {E2B2BAC0-D55C-45DB-8CB3-8CEBA86FB547} + EndGlobalSection +EndGlobal diff --git a/csharp/extractor/Semmle.Autobuild.Tests/BuildScripts.cs b/csharp/extractor/Semmle.Autobuild.Tests/BuildScripts.cs new file mode 100644 index 00000000000..d49493c5548 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild.Tests/BuildScripts.cs @@ -0,0 +1,883 @@ +using Xunit; +using Semmle.Autobuild; +using System.Collections.Generic; +using System; +using System.Linq; +using Microsoft.Build.Construction; + +namespace Semmle.Extraction.Tests +{ + ///

+ /// Test class to script Autobuilder scenarios. + /// For most methods, it uses two fields: + /// - an IList to capture the the arguments passed to it + /// - an IDictionary of possible return values. + /// + class TestActions : IBuildActions + { + /// + /// List of strings passed to FileDelete. + /// + public IList FileDeleteIn = new List(); + + void IBuildActions.FileDelete(string file) + { + FileDeleteIn.Add(file); + } + + public IList FileExistsIn = new List(); + public IDictionary FileExists = new Dictionary(); + + bool IBuildActions.FileExists(string file) + { + FileExistsIn.Add(file); + if (FileExists.TryGetValue(file, out var ret)) + return ret; + throw new ArgumentException("Missing FileExists " + file); + } + + public IList RunProcessIn = new List(); + public IDictionary RunProcess = new Dictionary(); + public IDictionary RunProcessOut = new Dictionary(); + public IDictionary RunProcessWorkingDirectory = new Dictionary(); + + int IBuildActions.RunProcess(string cmd, string args, string workingDirectory, IDictionary env, out IList stdOut) + { + var pattern = cmd + " " + args; + RunProcessIn.Add(pattern); + if (RunProcessOut.TryGetValue(pattern, out var str)) + stdOut = str.Split("\n"); + else + throw new ArgumentException("Missing RunProcessOut " + pattern); + RunProcessWorkingDirectory.TryGetValue(pattern, out var wd); + if (wd != workingDirectory) + throw new ArgumentException("Missing RunProcessWorkingDirectory " + pattern); + if (RunProcess.TryGetValue(pattern, out var ret)) + return ret; + throw new ArgumentException("Missing RunProcess " + pattern); + } + + int IBuildActions.RunProcess(string cmd, string args, string workingDirectory, IDictionary env) + { + var pattern = cmd + " " + args; + RunProcessIn.Add(pattern); + RunProcessWorkingDirectory.TryGetValue(pattern, out var wd); + if (wd != workingDirectory) + throw new ArgumentException("Missing RunProcessWorkingDirectory " + pattern); + if (RunProcess.TryGetValue(pattern, out var ret)) + return ret; + throw new ArgumentException("Missing RunProcess " + pattern); + } + + public IList DirectoryDeleteIn = new List(); + + void IBuildActions.DirectoryDelete(string dir, bool recursive) + { + DirectoryDeleteIn.Add(dir); + } + + public IDictionary DirectoryExists = new Dictionary(); + public IList DirectoryExistsIn = new List(); + + bool IBuildActions.DirectoryExists(string dir) + { + DirectoryExistsIn.Add(dir); + if (DirectoryExists.TryGetValue(dir, out var ret)) + return ret; + throw new ArgumentException("Missing DirectoryExists " + dir); + } + + public IDictionary GetEnvironmentVariable = new Dictionary(); + + string IBuildActions.GetEnvironmentVariable(string name) + { + if (GetEnvironmentVariable.TryGetValue(name, out var ret)) + return ret; + throw new ArgumentException("Missing GetEnvironmentVariable " + name); + } + + public string GetCurrentDirectory; + + string IBuildActions.GetCurrentDirectory() + { + return GetCurrentDirectory; + } + + public IDictionary EnumerateFiles = new Dictionary(); + + IEnumerable IBuildActions.EnumerateFiles(string dir) + { + if (EnumerateFiles.TryGetValue(dir, out var str)) + return str.Split("\n"); + throw new ArgumentException("Missing EnumerateFiles " + dir); + } + + public IDictionary EnumerateDirectories = new Dictionary(); + + IEnumerable IBuildActions.EnumerateDirectories(string dir) + { + if (EnumerateDirectories.TryGetValue(dir, out var str)) + return string.IsNullOrEmpty(str) ? Enumerable.Empty() : str.Split("\n"); + throw new ArgumentException("Missing EnumerateDirectories " + dir); + } + + public bool IsWindows; + + bool IBuildActions.IsWindows() => IsWindows; + + string IBuildActions.PathCombine(params string[] parts) + { + return string.Join('\\', parts); + } + + void IBuildActions.WriteAllText(string filename, string contents) + { + } + } + + /// + /// A fake solution to build. + /// + class TestSolution : ISolution + { + public IEnumerable Projects => throw new NotImplementedException(); + + public IEnumerable Configurations => throw new NotImplementedException(); + + public string DefaultConfigurationName => "Release"; + + public string DefaultPlatformName => "x86"; + + public string Path { get; set; } + + public int ProjectCount => throw new NotImplementedException(); + + public Version ToolsVersion => new Version("14.0"); + + public TestSolution(string path) + { + Path = path; + } + } + + public class BuildScriptTests + { + TestActions Actions = new TestActions(); + + // Records the arguments passed to StartCallback. + IList StartCallbackIn = new List(); + + void StartCallback(string s) + { + StartCallbackIn.Add(s); + } + + // Records the arguments passed to EndCallback + IList EndCallbackIn = new List(); + IList EndCallbackReturn = new List(); + + void EndCallback(int ret, string s) + { + EndCallbackReturn.Add(ret); + EndCallbackIn.Add(s); + } + + [Fact] + public void TestBuildCommand() + { + var cmd = BuildScript.Create("abc", "def ghi", null, null); + + Actions.RunProcess["abc def ghi"] = 1; + cmd.Run(Actions, StartCallback, EndCallback); + Assert.Equal("abc def ghi", Actions.RunProcessIn[0]); + Assert.Equal("abc def ghi", StartCallbackIn[0]); + Assert.Equal("", EndCallbackIn[0]); + Assert.Equal(1, EndCallbackReturn[0]); + } + + [Fact] + public void TestAnd1() + { + var cmd = BuildScript.Create("abc", "def ghi", null, null) & BuildScript.Create("odasa", null, null, null); + + Actions.RunProcess["abc def ghi"] = 1; + cmd.Run(Actions, StartCallback, EndCallback); + + Assert.Equal("abc def ghi", Actions.RunProcessIn[0]); + Assert.Equal("abc def ghi", StartCallbackIn[0]); + Assert.Equal("", EndCallbackIn[0]); + Assert.Equal(1, EndCallbackReturn[0]); + } + + [Fact] + public void TestAnd2() + { + var cmd = BuildScript.Create("odasa", null, null, null) & BuildScript.Create("abc", "def ghi", null, null); + + Actions.RunProcess["abc def ghi"] = 1; + Actions.RunProcess["odasa "] = 0; + cmd.Run(Actions, StartCallback, EndCallback); + + Assert.Equal("odasa ", Actions.RunProcessIn[0]); + Assert.Equal("odasa ", StartCallbackIn[0]); + Assert.Equal("", EndCallbackIn[0]); + Assert.Equal(0, EndCallbackReturn[0]); + + Assert.Equal("abc def ghi", Actions.RunProcessIn[1]); + Assert.Equal("abc def ghi", StartCallbackIn[1]); + Assert.Equal("", EndCallbackIn[1]); + Assert.Equal(1, EndCallbackReturn[1]); + } + + [Fact] + public void TestOr1() + { + var cmd = BuildScript.Create("odasa", null, null, null) | BuildScript.Create("abc", "def ghi", null, null); + + Actions.RunProcess["abc def ghi"] = 1; + Actions.RunProcess["odasa "] = 0; + cmd.Run(Actions, StartCallback, EndCallback); + + Assert.Equal("odasa ", Actions.RunProcessIn[0]); + Assert.Equal("odasa ", StartCallbackIn[0]); + Assert.Equal("", EndCallbackIn[0]); + Assert.Equal(0, EndCallbackReturn[0]); + Assert.Equal(1, EndCallbackReturn.Count); + } + + [Fact] + public void TestOr2() + { + var cmd = BuildScript.Create("abc", "def ghi", null, null) | BuildScript.Create("odasa", null, null, null); + + Actions.RunProcess["abc def ghi"] = 1; + Actions.RunProcess["odasa "] = 0; + cmd.Run(Actions, StartCallback, EndCallback); + + Assert.Equal("abc def ghi", Actions.RunProcessIn[0]); + Assert.Equal("abc def ghi", StartCallbackIn[0]); + Assert.Equal("", EndCallbackIn[0]); + Assert.Equal(1, EndCallbackReturn[0]); + + Assert.Equal("odasa ", Actions.RunProcessIn[1]); + Assert.Equal("odasa ", StartCallbackIn[1]); + Assert.Equal("", EndCallbackIn[1]); + Assert.Equal(0, EndCallbackReturn[1]); + } + + [Fact] + public void TestSuccess() + { + Assert.Equal(0, BuildScript.Success.Run(Actions, StartCallback, EndCallback)); + } + + [Fact] + public void TestFailure() + { + Assert.NotEqual(0, BuildScript.Failure.Run(Actions, StartCallback, EndCallback)); + } + + [Fact] + public void TestDeleteDirectorySuccess() + { + Actions.DirectoryExists["trap"] = true; + Assert.Equal(0, BuildScript.DeleteDirectory("trap").Run(Actions, StartCallback, EndCallback)); + Assert.Equal("trap", Actions.DirectoryDeleteIn[0]); + } + + [Fact] + public void TestDeleteDirectoryFailure() + { + Actions.DirectoryExists["trap"] = false; + Assert.NotEqual(0, BuildScript.DeleteDirectory("trap").Run(Actions, StartCallback, EndCallback)); + } + + [Fact] + public void TestDeleteFileSuccess() + { + Actions.FileExists["csharp.log"] = true; + Assert.Equal(0, BuildScript.DeleteFile("csharp.log").Run(Actions, StartCallback, EndCallback)); + Assert.Equal("csharp.log", Actions.FileExistsIn[0]); + Assert.Equal("csharp.log", Actions.FileDeleteIn[0]); + } + + [Fact] + public void TestDeleteFileFailure() + { + Actions.FileExists["csharp.log"] = false; + Assert.NotEqual(0, BuildScript.DeleteFile("csharp.log").Run(Actions, StartCallback, EndCallback)); + Assert.Equal("csharp.log", Actions.FileExistsIn[0]); + } + + [Fact] + public void TestTry() + { + Assert.Equal(0, BuildScript.Try(BuildScript.Failure).Run(Actions, StartCallback, EndCallback)); + } + + Autobuilder CreateAutoBuilder(string lgtmLanguage, bool isWindows, + string buildless=null, string solution=null, string buildCommand=null, string ignoreErrors=null, + string msBuildArguments=null, string msBuildPlatform=null, string msBuildConfiguration=null, string msBuildTarget=null, + string dotnetArguments=null, string dotnetVersion=null, string vsToolsVersion=null, + string nugetRestore=null, string allSolutions=null, + string cwd=@"C:\Project") + { + Actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa"; + Actions.GetEnvironmentVariable["SEMMLE_JAVA_HOME"] = @"C:\odasa\tools\java"; + Actions.GetEnvironmentVariable["LGTM_PROJECT_LANGUAGE"] = lgtmLanguage; + Actions.GetEnvironmentVariable["SEMMLE_PLATFORM_TOOLS"] = @"C:\odasa\tools"; + Actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion; + Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_ARGUMENTS"] = msBuildArguments; + Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_PLATFORM"] = msBuildPlatform; + Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_CONFIGURATION"] = msBuildConfiguration; + Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_TARGET"] = msBuildTarget; + Actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_ARGUMENTS"] = dotnetArguments; + Actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_VERSION"] = dotnetVersion; + Actions.GetEnvironmentVariable["LGTM_INDEX_BUILD_COMMAND"] = buildCommand; + Actions.GetEnvironmentVariable["LGTM_INDEX_SOLUTION"] = solution; + Actions.GetEnvironmentVariable["LGTM_INDEX_IGNORE_ERRORS"] = ignoreErrors; + Actions.GetEnvironmentVariable["LGTM_INDEX_BUILDLESS"] = buildless; + Actions.GetEnvironmentVariable["LGTM_INDEX_ALL_SOLUTIONS"] = allSolutions; + Actions.GetEnvironmentVariable["LGTM_INDEX_NUGET_RESTORE"] = nugetRestore; + Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = isWindows ? @"C:\Program Files (x86)" : null; + Actions.GetCurrentDirectory = cwd; + Actions.IsWindows = isWindows; + + var options = new AutobuildOptions(); + options.ReadEnvironment(Actions); + return new Autobuilder(Actions, options); + } + + [Fact] + public void TestDefaultCSharpAutoBuilder() + { + Actions.RunProcess["cmd.exe /C dotnet --info"] = 0; + Actions.RunProcess["cmd.exe /C dotnet clean"] = 0; + Actions.RunProcess["cmd.exe /C dotnet restore"] = 0; + Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false"] = 0; + Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0; + Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --xml --extensions config"] = 0; + Actions.FileExists["csharp.log"] = true; + Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null; + Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null; + Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\nbar.cs"; + Actions.EnumerateDirectories[@"C:\Project"] = ""; + + var autobuilder = CreateAutoBuilder("csharp", true); + TestAutobuilderScript(autobuilder, 0, 6); + } + + [Fact] + public void TestLinuxCSharpAutoBuilder() + { + Actions.RunProcess["dotnet --info"] = 0; + Actions.RunProcess["dotnet clean"] = 0; + Actions.RunProcess["dotnet restore"] = 0; + Actions.RunProcess[@"C:\odasa\tools\odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false"] = 0; + Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0; + Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config"] = 0; + Actions.FileExists["csharp.log"] = true; + Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null; + Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null; + Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.cs"; + Actions.EnumerateDirectories[@"C:\Project"] = ""; + + var autobuilder = CreateAutoBuilder("csharp", false); + TestAutobuilderScript(autobuilder, 0, 6); + } + + [Fact] + public void TestLinuxCSharpAutoBuilderExtractorFailed() + { + Actions.RunProcess["dotnet --info"] = 0; + Actions.RunProcess["dotnet clean"] = 0; + Actions.RunProcess["dotnet restore"] = 0; + Actions.RunProcess[@"C:\odasa\tools\odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false"] = 0; + Actions.FileExists["csharp.log"] = false; + Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null; + Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null; + Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.cs"; + Actions.EnumerateDirectories[@"C:\Project"] = ""; + + var autobuilder = CreateAutoBuilder("csharp", false); + TestAutobuilderScript(autobuilder, 1, 4); + } + + + [Fact] + public void TestDefaultCppAutobuilder() + { + Actions.EnumerateFiles[@"C:\Project"] = ""; + Actions.EnumerateDirectories[@"C:\Project"] = ""; + + var autobuilder = CreateAutoBuilder("cpp", true); + var script = autobuilder.GetBuildScript(); + + // Fails due to no solutions present. + Assert.NotEqual(0, script.Run(Actions, StartCallback, EndCallback)); + } + + [Fact] + public void TestCppAutobuilderSuccess() + { + Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\csharp\nuget\nuget.exe restore C:\Project\test.sln"] = 1; + Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program Files ^(x86^)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat^"" && C:\odasa\tools\odasa index --auto msbuild C:\Project\test.sln /p:UseSharedCompilation=false /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"" /p:MvcBuildViews=true"] = 0; + Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = ""; + Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = 1; + Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = 0; + Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = ""; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = true; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = true; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = true; + Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.slx"; + Actions.EnumerateDirectories[@"C:\Project"] = ""; + + var autobuilder = CreateAutoBuilder("cpp", true); + var solution = new TestSolution(@"C:\Project\test.sln"); + autobuilder.SolutionsToBuild.Add(solution); + TestAutobuilderScript(autobuilder, 0, 2); + } + + [Fact] + public void TestVsWhereSucceeded() + { + Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = @"C:\Program Files (x86)"; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = true; + Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = 0; + Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = "C:\\VS1\nC:\\VS2"; + Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = "10.0\n11.0"; + Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = 0; + + var candidates = BuildTools.GetCandidateVcVarsFiles(Actions).ToArray(); + Assert.Equal("C:\\VS1\\VC\\vcvarsall.bat", candidates[0].Path); + Assert.Equal(10, candidates[0].ToolsVersion); + Assert.Equal("C:\\VS2\\VC\\vcvarsall.bat", candidates[1].Path); + Assert.Equal(11, candidates[1].ToolsVersion); + } + + [Fact] + public void TestVsWhereNotExist() + { + Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = @"C:\Program Files (x86)"; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false; + + var candidates = BuildTools.GetCandidateVcVarsFiles(Actions).ToArray(); + Assert.Equal(4, candidates.Length); + } + + [Fact] + public void TestVcVarsAllBatFiles() + { + Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = @"C:\Program Files (x86)"; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = true; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = false; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = true; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = false; + + var vcvarsfiles = BuildTools.VcVarsAllBatFiles(Actions).ToArray(); + Assert.Equal(2, vcvarsfiles.Length); + } + + [Fact] + public void TestLinuxBuildlessExtractionSuccess() + { + Actions.RunProcess[@"C:\odasa\tools\csharp\Semmle.Extraction.CSharp.Standalone --references:."] = 0; + Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0; + Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config"] = 0; + Actions.FileExists["csharp.log"] = true; + Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null; + Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null; + Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln"; + Actions.EnumerateDirectories[@"C:\Project"] = ""; + + var autobuilder = CreateAutoBuilder("csharp", false, buildless:"true"); + TestAutobuilderScript(autobuilder, 0, 3); + } + + [Fact] + public void TestLinuxBuildlessExtractionFailed() + { + Actions.RunProcess[@"C:\odasa\tools\csharp\Semmle.Extraction.CSharp.Standalone --references:."] = 10; + Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0; + Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config"] = 0; + Actions.FileExists["csharp.log"] = true; + Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null; + Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null; + Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln"; + Actions.EnumerateDirectories[@"C:\Project"] = ""; + + var autobuilder = CreateAutoBuilder("csharp", false, buildless: "true"); + TestAutobuilderScript(autobuilder, 10, 1); + } + + [Fact] + public void TestLinuxBuildlessExtractionSolution() + { + Actions.RunProcess[@"C:\odasa\tools\csharp\Semmle.Extraction.CSharp.Standalone foo.sln --references:."] = 0; + Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0; + Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config"] = 0; + Actions.FileExists["csharp.log"] = true; + Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null; + Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null; + Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln"; + Actions.EnumerateDirectories[@"C:\Project"] = ""; + + var autobuilder = CreateAutoBuilder("csharp", false, buildless: "true", solution: "foo.sln"); + TestAutobuilderScript(autobuilder, 0, 3); + } + + void SkipVsWhere() + { + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = false; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = false; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = false; + } + + void TestAutobuilderScript(Autobuilder autobuilder, int expectedOutput, int commandsRun) + { + Assert.Equal(expectedOutput, autobuilder.GetBuildScript().Run(Actions, StartCallback, EndCallback)); + + // Check expected commands actually ran + Assert.Equal(commandsRun, StartCallbackIn.Count); + Assert.Equal(commandsRun, EndCallbackIn.Count); + Assert.Equal(commandsRun, EndCallbackReturn.Count); + + var action = Actions.RunProcess.GetEnumerator(); + for(int cmd=0; cmd + + + Exe + netcoreapp2.0 + false + + + + + + + + + + + + diff --git a/csharp/extractor/Semmle.Autobuild/AspBuildRule.cs b/csharp/extractor/Semmle.Autobuild/AspBuildRule.cs new file mode 100644 index 00000000000..7765e94f5e1 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/AspBuildRule.cs @@ -0,0 +1,20 @@ +using System.IO; + +namespace Semmle.Autobuild +{ + /// + /// ASP extraction. + /// + class AspBuildRule : IBuildRule + { + public BuildScript Analyse(Autobuilder builder) + { + var command = new CommandBuilder(builder.Actions). + RunCommand(builder.Actions.PathCombine(builder.SemmleJavaHome, "bin", "java")). + Argument("-jar"). + QuoteArgument(builder.Actions.PathCombine(builder.SemmleDist, "tools", "extractor-asp.jar")). + Argument("."); + return command.Script; + } + } +} diff --git a/csharp/extractor/Semmle.Autobuild/AutobuildOptions.cs b/csharp/extractor/Semmle.Autobuild/AutobuildOptions.cs new file mode 100644 index 00000000000..53ecf697721 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/AutobuildOptions.cs @@ -0,0 +1,103 @@ +using System; +using System.Linq; + +namespace Semmle.Autobuild +{ + /// + /// Encapsulates build options. + /// + public class AutobuildOptions + { + public readonly int SearchDepth = 3; + public string RootDirectory = null; + static readonly string prefix = "LGTM_INDEX_"; + + public string VsToolsVersion; + public string MsBuildArguments; + public string MsBuildPlatform; + public string MsBuildConfiguration; + public string MsBuildTarget; + public string DotNetArguments; + public string DotNetVersion; + public string BuildCommand; + public string[] Solution; + + public bool IgnoreErrors; + public bool Buildless; + public bool AllSolutions; + public bool NugetRestore; + + public Language Language; + + /// + /// Reads options from environment variables. + /// Throws ArgumentOutOfRangeException for invalid arguments. + /// + public void ReadEnvironment(IBuildActions actions) + { + RootDirectory = actions.GetCurrentDirectory(); + VsToolsVersion = actions.GetEnvironmentVariable(prefix + "VSTOOLS_VERSION"); + MsBuildArguments = actions.GetEnvironmentVariable(prefix + "MSBUILD_ARGUMENTS"); + MsBuildPlatform = actions.GetEnvironmentVariable(prefix + "MSBUILD_PLATFORM"); + MsBuildConfiguration = actions.GetEnvironmentVariable(prefix + "MSBUILD_CONFIGURATION"); + MsBuildTarget = actions.GetEnvironmentVariable(prefix + "MSBUILD_TARGET"); + DotNetArguments = actions.GetEnvironmentVariable(prefix + "DOTNET_ARGUMENTS"); + DotNetVersion = actions.GetEnvironmentVariable(prefix + "DOTNET_VERSION"); + BuildCommand = actions.GetEnvironmentVariable(prefix + "BUILD_COMMAND"); + Solution = actions.GetEnvironmentVariable(prefix + "SOLUTION").AsList(new string[0]); + + IgnoreErrors = actions.GetEnvironmentVariable(prefix + "IGNORE_ERRORS").AsBool("ignore_errors", false); + Buildless = actions.GetEnvironmentVariable(prefix + "BUILDLESS").AsBool("buildless", false); + AllSolutions = actions.GetEnvironmentVariable(prefix + "ALL_SOLUTIONS").AsBool("all_solutions", false); + NugetRestore = actions.GetEnvironmentVariable(prefix + "NUGET_RESTORE").AsBool("nuget_restore", true); + + Language = actions.GetEnvironmentVariable("LGTM_PROJECT_LANGUAGE").AsLanguage(); + } + } + + public static class OptionsExtensions + { + public static bool AsBool(this string value, string param, bool defaultValue) + { + if (value == null) return defaultValue; + switch (value.ToLower()) + { + case "on": + case "yes": + case "true": + case "enabled": + return true; + case "off": + case "no": + case "false": + case "disabled": + return false; + default: + throw new ArgumentOutOfRangeException(param, value, "The Boolean value is invalid."); + } + } + + public static Language AsLanguage(this string key) + { + switch (key) + { + case null: + throw new ArgumentException("Environment variable required: LGTM_PROJECT_LANGUAGE"); + case "csharp": + return Language.CSharp; + case "cpp": + return Language.Cpp; + default: + throw new ArgumentException("Language key not understood: '" + key + "'"); + } + } + + public static string[] AsList(this string value, string[] defaultValue) + { + if (value == null) + return defaultValue; + + return value.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries).ToArray(); + } + } +} diff --git a/csharp/extractor/Semmle.Autobuild/Autobuilder.cs b/csharp/extractor/Semmle.Autobuild/Autobuilder.cs new file mode 100644 index 00000000000..3edcb802ffe --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/Autobuilder.cs @@ -0,0 +1,352 @@ +using Semmle.Extraction.CSharp; +using Semmle.Util.Logging; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace Semmle.Autobuild +{ + /// + /// A build rule analyses the files in "builder" and outputs a build script. + /// + interface IBuildRule + { + /// + /// Analyse the files and produce a build script. + /// + /// The files and options relating to the build. + BuildScript Analyse(Autobuilder builder); + } + + /// + /// Main application logic, containing all data + /// gathered from the project and filesystem. + /// + /// The overall design is intended to be extensible so that in theory, + /// it should be possible to add new build rules without touching this code. + /// + public class Autobuilder + { + /// + /// Full file paths of files found in the project directory. + /// + public IEnumerable Paths => pathsLazy.Value; + readonly Lazy> pathsLazy; + + /// + /// Gets a list of paths matching a set of extensions + /// (including the "."). + /// + /// The extensions to find. + /// The files matching the extension. + public IEnumerable GetExtensions(params string[] extensions) => + Paths.Where(p => extensions.Contains(Path.GetExtension(p))); + + /// + /// Gets all paths matching a particular filename. + /// + /// The filename to find. + /// Possibly empty sequence of paths with the given filename. + public IEnumerable GetFilename(string name) => Paths.Where(p => Path.GetFileName(p) == name); + + /// + /// Holds if a given path, relative to the root of the source directory + /// was found. + /// + /// The relative path. + /// True iff the path was found. + public bool HasRelativePath(string path) => HasPath(Actions.PathCombine(RootDirectory, path)); + + /// + /// List of solution files to build. + /// + public IList SolutionsToBuild => solutionsToBuildLazy.Value; + readonly Lazy> solutionsToBuildLazy; + + /// + /// Holds if a given path was found. + /// + /// The path of the file. + /// True iff the path was found. + public bool HasPath(string path) => Paths.Any(p => path == p); + + void FindFiles(string dir, int depth, IList results) + { + foreach (var f in Actions.EnumerateFiles(dir)) + { + results.Add(f); + } + + if (depth > 1) + { + foreach (var d in Actions.EnumerateDirectories(dir)) + { + FindFiles(d, depth - 1, results); + } + } + } + + /// + /// The root of the source directory. + /// + string RootDirectory => Options.RootDirectory; + + /// + /// Gets the supplied build configuration. + /// + public AutobuildOptions Options { get; } + + /// + /// The set of build actions used during the autobuilder. + /// Could be real system operations, or a stub for testing. + /// + public IBuildActions Actions { get; } + + /// + /// Find all the relevant files and picks the best + /// solution file and tools. + /// + /// The command line options. + public Autobuilder(IBuildActions actions, AutobuildOptions options) + { + Actions = actions; + Options = options; + + pathsLazy = new Lazy>(() => + { + var files = new List(); + FindFiles(options.RootDirectory, options.SearchDepth, files); + return files. + OrderBy(s => s.Count(c => c == Path.DirectorySeparatorChar)). + ThenBy(s => Path.GetFileName(s).Length). + ToArray(); + }); + + solutionsToBuildLazy = new Lazy>(() => + { + if (options.Solution.Any()) + { + var ret = new List(); + foreach (var solution in options.Solution) + { + if (actions.FileExists(solution)) + ret.Add(new Solution(this, solution)); + else + Log(Severity.Error, "The specified solution file {0} was not found", solution); + } + return ret; + } + + var solutions = GetExtensions(".sln"). + Select(s => new Solution(this, s)). + Where(s => s.ProjectCount > 0). + OrderByDescending(s => s.ProjectCount). + ThenBy(s => s.Path.Length). + ToArray(); + + foreach (var sln in solutions) + { + Log(Severity.Info, $"Found {sln.Path} with {sln.ProjectCount} {this.Options.Language} projects, version {sln.ToolsVersion}, config {string.Join(" ", sln.Configurations.Select(c => c.FullName))}"); + } + + return new List(options.AllSolutions ? + solutions : + solutions.Take(1)); + }); + + SemmleDist = Actions.GetEnvironmentVariable("SEMMLE_DIST"); + + SemmleJavaHome = Actions.GetEnvironmentVariable("SEMMLE_JAVA_HOME"); + + SemmlePlatformTools = Actions.GetEnvironmentVariable("SEMMLE_PLATFORM_TOOLS"); + + if (SemmleDist == null) + Log(Severity.Error, "The environment variable SEMMLE_DIST has not been set."); + } + + readonly ILogger logger = new ConsoleLogger(Verbosity.Info); + + /// + /// Log a given build event to the console. + /// + /// The format string. + /// Inserts to the format string. + public void Log(Severity severity, string format, params object[] args) + { + logger.Log(severity, format, args); + } + + /// + /// Attempt to build this project. + /// + /// The exit code, 0 for success and non-zero for failures. + public int AttemptBuild() + { + Log(Severity.Info, $"Working directory: {Options.RootDirectory}"); + + var script = GetBuildScript(); + if (Options.IgnoreErrors) + script |= BuildScript.Success; + + void startCallback(string s) => Log(Severity.Info, $"\nRunning {s}"); + void exitCallback(int ret, string msg) => Log(Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}"); + return script.Run(Actions, startCallback, exitCallback); + } + + /// + /// Returns the build script to use for this project. + /// + public BuildScript GetBuildScript() + { + var isCSharp = Options.Language == Language.CSharp; + return isCSharp ? GetCSharpBuildScript() : GetCppBuildScript(); + } + + BuildScript GetCSharpBuildScript() + { + /// + /// A script that checks that the C# extractor has been executed. + /// + BuildScript CheckExtractorRun(bool warnOnFailure) => + BuildScript.Create(actions => + { + if (actions.FileExists(Extractor.GetCSharpLogPath())) + return 0; + + if (warnOnFailure) + Log(Severity.Error, "No C# code detected during build."); + + return 1; + }); + + var attempt = BuildScript.Failure; + switch (GetCSharpBuildStrategy()) + { + case CSharpBuildStrategy.CustomBuildCommand: + attempt = new BuildCommandRule().Analyse(this) & CheckExtractorRun(true); + break; + case CSharpBuildStrategy.Buildless: + // No need to check that the extractor has been executed in buildless mode + attempt = new StandaloneBuildRule().Analyse(this); + break; + case CSharpBuildStrategy.MSBuild: + attempt = new MsBuildRule().Analyse(this) & CheckExtractorRun(true); + break; + case CSharpBuildStrategy.DotNet: + attempt = new DotNetRule().Analyse(this) & CheckExtractorRun(true); + break; + case CSharpBuildStrategy.Auto: + var cleanTrapFolder = + BuildScript.DeleteDirectory(Actions.GetEnvironmentVariable("TRAP_FOLDER")); + var cleanSourceArchive = + BuildScript.DeleteDirectory(Actions.GetEnvironmentVariable("SOURCE_ARCHIVE")); + var cleanExtractorLog = + BuildScript.DeleteFile(Extractor.GetCSharpLogPath()); + var attemptExtractorCleanup = + BuildScript.Try(cleanTrapFolder) & + BuildScript.Try(cleanSourceArchive) & + BuildScript.Try(cleanExtractorLog); + + /// + /// Execute script `s` and check that the C# extractor has been executed. + /// If either fails, attempt to cleanup any artifacts produced by the extractor, + /// and exit with code 1, in order to proceed to the next attempt. + /// + BuildScript IntermediateAttempt(BuildScript s) => + (s & CheckExtractorRun(false)) | + (attemptExtractorCleanup & BuildScript.Failure); + + attempt = + // First try .NET Core + IntermediateAttempt(new DotNetRule().Analyse(this)) | + // Then MSBuild + (() => IntermediateAttempt(new MsBuildRule().Analyse(this))) | + // And finally look for a script that might be a build script + (() => new BuildCommandAutoRule().Analyse(this) & CheckExtractorRun(true)) | + // All attempts failed: print message + AutobuildFailure(); + break; + } + + return + attempt & + (() => new AspBuildRule().Analyse(this)) & + (() => new XmlBuildRule().Analyse(this)); + } + + /// + /// Gets the build strategy that the autobuilder should apply, based on the + /// options in the `lgtm.yml` file. + /// + CSharpBuildStrategy GetCSharpBuildStrategy() + { + if (Options.BuildCommand != null) + return CSharpBuildStrategy.CustomBuildCommand; + + if (Options.Buildless) + return CSharpBuildStrategy.Buildless; + + if (Options.MsBuildArguments != null + || Options.MsBuildConfiguration != null + || Options.MsBuildPlatform != null + || Options.MsBuildTarget != null) + return CSharpBuildStrategy.MSBuild; + + if (Options.DotNetArguments != null || Options.DotNetVersion != null) + return CSharpBuildStrategy.DotNet; + + return CSharpBuildStrategy.Auto; + } + + enum CSharpBuildStrategy + { + CustomBuildCommand, + Buildless, + MSBuild, + DotNet, + Auto + } + + BuildScript GetCppBuildScript() + { + if (Options.BuildCommand != null) + return new BuildCommandRule().Analyse(this); + + return + // First try MSBuild + new MsBuildRule().Analyse(this) | + // Then look for a script that might be a build script + (() => new BuildCommandAutoRule().Analyse(this)) | + // All attempts failed: print message + AutobuildFailure(); + } + + BuildScript AutobuildFailure() => + BuildScript.Create(actions => + { + Log(Severity.Error, "Could not auto-detect a suitable build method"); + return 1; + }); + + /// + /// Value of SEMMLE_DIST environment variable. + /// + public string SemmleDist { get; private set; } + + /// + /// Value of SEMMLE_JAVA_HOME environment variable. + /// + public string SemmleJavaHome { get; private set; } + + /// + /// Value of SEMMLE_PLATFORM_TOOLS environment variable. + /// + public string SemmlePlatformTools { get; private set; } + + /// + /// The absolute path of the odasa executable. + /// + public string Odasa => SemmleDist == null ? null : Actions.PathCombine(SemmleDist, "tools", "odasa"); + } +} diff --git a/csharp/extractor/Semmle.Autobuild/BuildActions.cs b/csharp/extractor/Semmle.Autobuild/BuildActions.cs new file mode 100644 index 00000000000..906d6701c1a --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/BuildActions.cs @@ -0,0 +1,176 @@ +using Semmle.Util; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; + +namespace Semmle.Autobuild +{ + /// + /// Wrapper around system calls so that the build scripts can be unit-tested. + /// + public interface IBuildActions + { + /// + /// Runs a process and captures its output. + /// + /// The exe to run. + /// The other command line arguments. + /// The working directory (null for current directory). + /// Additional environment variables. + /// The lines of stdout. + /// The process exit code. + int RunProcess(string exe, string args, string workingDirectory, IDictionary env, out IList stdOut); + + /// + /// Runs a process but does not capture its output. + /// + /// The exe to run. + /// The other command line arguments. + /// The working directory (null for current directory). + /// Additional environment variables. + /// The process exit code. + int RunProcess(string exe, string args, string workingDirectory, IDictionary env); + + /// + /// Tests whether a file exists, File.Exists(). + /// + /// The filename. + /// True iff the file exists. + bool FileExists(string file); + + /// + /// Tests whether a directory exists, Directory.Exists(). + /// + /// The directory name. + /// True iff the directory exists. + bool DirectoryExists(string dir); + + /// + /// Deletes a file, File.Delete(). + /// + /// The filename. + void FileDelete(string file); + + /// + /// Deletes a directory, Directory.Delete(). + /// + void DirectoryDelete(string dir, bool recursive); + + /// + /// Gets an environment variable, Environment.GetEnvironmentVariable(). + /// + /// The name of the variable. + /// The string value, or null if the variable is not defined. + string GetEnvironmentVariable(string name); + + /// + /// Gets the current directory, Directory.GetCurrentDirectory(). + /// + /// The current directory. + string GetCurrentDirectory(); + + /// + /// Enumerates files in a directory, Directory.EnumerateFiles(). + /// + /// The directory to enumerate. + /// A list of filenames, or an empty list. + IEnumerable EnumerateFiles(string dir); + + /// + /// Enumerates the directories in a directory, Directory.EnumerateDirectories(). + /// + /// The directory to enumerate. + /// List of subdirectories, or empty list. + IEnumerable EnumerateDirectories(string dir); + + /// + /// True if we are running on Windows. + /// + bool IsWindows(); + + /// + /// Combine path segments, Path.Combine(). + /// + /// The parts of the path. + /// The combined path. + string PathCombine(params string[] parts); + + /// + /// Writes contents to file, File.WriteAllText(). + /// + /// The filename. + /// The text. + void WriteAllText(string filename, string contents); + } + + /// + /// An implementation of IBuildActions that actually performs the requested operations. + /// + class SystemBuildActions : IBuildActions + { + void IBuildActions.FileDelete(string file) => File.Delete(file); + + bool IBuildActions.FileExists(string file) => File.Exists(file); + + ProcessStartInfo GetProcessStartInfo(string exe, string arguments, string workingDirectory, IDictionary environment, bool redirectStandardOutput) + { + var pi = new ProcessStartInfo(exe, arguments) + { + UseShellExecute = false, + RedirectStandardOutput = redirectStandardOutput + }; + if (workingDirectory != null) + pi.WorkingDirectory = workingDirectory; + + // Environment variables can only be used when not redirecting stdout + if (!redirectStandardOutput) + { + pi.Environment["UseSharedCompilation"] = "false"; + if (environment != null) + environment.ForEach(kvp => pi.Environment[kvp.Key] = kvp.Value); + } + return pi; + } + + int IBuildActions.RunProcess(string cmd, string args, string workingDirectory, IDictionary environment) + { + var pi = GetProcessStartInfo(cmd, args, workingDirectory, environment, false); + using (var p = Process.Start(pi)) + { + p.WaitForExit(); + return p.ExitCode; + } + } + + int IBuildActions.RunProcess(string cmd, string args, string workingDirectory, IDictionary environment, out IList stdOut) + { + var pi = GetProcessStartInfo(cmd, args, workingDirectory, environment, true); + return pi.ReadOutput(out stdOut); + } + + void IBuildActions.DirectoryDelete(string dir, bool recursive) => Directory.Delete(dir, recursive); + + bool IBuildActions.DirectoryExists(string dir) => Directory.Exists(dir); + + string IBuildActions.GetEnvironmentVariable(string name) => Environment.GetEnvironmentVariable(name); + + string IBuildActions.GetCurrentDirectory() => Directory.GetCurrentDirectory(); + + IEnumerable IBuildActions.EnumerateFiles(string dir) => Directory.EnumerateFiles(dir); + + IEnumerable IBuildActions.EnumerateDirectories(string dir) => Directory.EnumerateDirectories(dir); + + bool IBuildActions.IsWindows() => Win32.IsWindows(); + + string IBuildActions.PathCombine(params string[] parts) => Path.Combine(parts); + + void IBuildActions.WriteAllText(string filename, string contents) => File.WriteAllText(filename, contents); + + private SystemBuildActions() + { + } + + public static readonly IBuildActions Instance = new SystemBuildActions(); + } +} diff --git a/csharp/extractor/Semmle.Autobuild/BuildCommandAutoRule.cs b/csharp/extractor/Semmle.Autobuild/BuildCommandAutoRule.cs new file mode 100644 index 00000000000..627651eaefe --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/BuildCommandAutoRule.cs @@ -0,0 +1,61 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Semmle.Util; +using Semmle.Util.Logging; + +namespace Semmle.Autobuild +{ + /// + /// Auto-detection of build scripts. + /// + class BuildCommandAutoRule : IBuildRule + { + readonly IEnumerable winExtensions = new List { + ".bat", + ".cmd", + ".exe" + }; + + readonly IEnumerable linuxExtensions = new List { + "", + ".sh" + }; + + readonly IEnumerable buildScripts = new List { + "build" + }; + + public BuildScript Analyse(Autobuilder builder) + { + builder.Log(Severity.Info, "Attempting to locate build script"); + + var extensions = builder.Actions.IsWindows() ? winExtensions : linuxExtensions; + var scripts = buildScripts.SelectMany(s => extensions.Select(e => s + e)); + var scriptPath = builder.Paths.Where(p => scripts.Any(p.ToLower().EndsWith)).OrderBy(p => p.Length).FirstOrDefault(); + + if (scriptPath == null) + return BuildScript.Failure; + + var chmod = new CommandBuilder(builder.Actions); + chmod.RunCommand("/bin/chmod", $"u+x {scriptPath}"); + var chmodScript = builder.Actions.IsWindows() ? BuildScript.Success : BuildScript.Try(chmod.Script); + + var path = Path.GetDirectoryName(scriptPath); + + // A specific .NET Core version may be required + return chmodScript & DotNetRule.WithDotNet(builder, dotNet => + { + var command = new CommandBuilder(builder.Actions, path, dotNet?.Environment); + + // A specific Visual Studio version may be required + var vsTools = MsBuildRule.GetVcVarsBatFile(builder); + if (vsTools != null) + command.CallBatFile(vsTools.Path); + + command.IndexCommand(builder.Odasa, scriptPath); + return command.Script; + }); + } + } +} diff --git a/csharp/extractor/Semmle.Autobuild/BuildCommandRule.cs b/csharp/extractor/Semmle.Autobuild/BuildCommandRule.cs new file mode 100644 index 00000000000..2dd3a144222 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/BuildCommandRule.cs @@ -0,0 +1,28 @@ +namespace Semmle.Autobuild +{ + /// + /// Execute the build_command rule. + /// + class BuildCommandRule : IBuildRule + { + public BuildScript Analyse(Autobuilder builder) + { + if (builder.Options.BuildCommand == null) + return BuildScript.Failure; + + // Custom build commands may require a specific .NET Core version + return DotNetRule.WithDotNet(builder, dotNet => + { + var command = new CommandBuilder(builder.Actions, null, dotNet?.Environment); + + // Custom build commands may require a specific Visual Studio version + var vsTools = MsBuildRule.GetVcVarsBatFile(builder); + if (vsTools != null) + command.CallBatFile(vsTools.Path); + command.IndexCommand(builder.Odasa, builder.Options.BuildCommand); + + return command.Script; + }); + } + } +} diff --git a/csharp/extractor/Semmle.Autobuild/BuildScript.cs b/csharp/extractor/Semmle.Autobuild/BuildScript.cs new file mode 100644 index 00000000000..d36e98a0db5 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/BuildScript.cs @@ -0,0 +1,274 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using Semmle.Util; + +namespace Semmle.Autobuild +{ + /// + /// A build script. + /// + public abstract class BuildScript + { + /// + /// Run this build script. + /// + /// + /// The interface used to implement the build actions. + /// + /// + /// A call back that is called every time a new process is started. The + /// argument to the call back is a textual representation of the process. + /// + /// + /// A call back that is called every time a new process exits. The first + /// argument to the call back is the exit code, and the second argument is + /// an exit message. + /// + /// The exit code from this build script. + public abstract int Run(IBuildActions actions, Action startCallback, Action exitCallBack); + + /// + /// Run this build command. + /// + /// + /// The interface used to implement the build actions. + /// + /// + /// A call back that is called every time a new process is started. The + /// argument to the call back is a textual representation of the process. + /// + /// + /// A call back that is called every time a new process exits. The first + /// argument to the call back is the exit code, and the second argument is + /// an exit message. + /// + /// Contents of standard out. + /// The exit code from this build script. + public abstract int Run(IBuildActions actions, Action startCallback, Action exitCallBack, out IList stdout); + + class BuildCommand : BuildScript + { + readonly string exe, arguments, workingDirectory; + readonly IDictionary environment; + + /// + /// Create a simple build command. + /// + /// The executable to run. + /// The arguments to the executable, or null. + /// The working directory (null for current directory). + /// Additional environment variables. + public BuildCommand(string exe, string argumentsOpt, string workingDirectory = null, IDictionary environment = null) + { + this.exe = exe; + this.arguments = argumentsOpt ?? ""; + this.workingDirectory = workingDirectory; + this.environment = environment; + } + + public override string ToString() => exe + " " + arguments; + + public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack) + { + startCallback(this.ToString()); + var ret = 1; + var retMessage = ""; + try + { + ret = actions.RunProcess(exe, arguments, workingDirectory, environment); + } + catch (Exception ex) + when (ex is System.ComponentModel.Win32Exception || ex is FileNotFoundException) + { + retMessage = ex.Message; + } + + exitCallBack(ret, retMessage); + return ret; + } + + public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack, out IList stdout) + { + startCallback(this.ToString()); + var ret = 1; + var retMessage = ""; + try + { + ret = actions.RunProcess(exe, arguments, workingDirectory, environment, out stdout); + } + catch (Exception ex) + when (ex is System.ComponentModel.Win32Exception || ex is FileNotFoundException) + { + retMessage = ex.Message; + stdout = new string[0]; + } + exitCallBack(ret, retMessage); + return ret; + } + + } + + class ReturnBuildCommand : BuildScript + { + readonly Func func; + public ReturnBuildCommand(Func func) + { + this.func = func; + } + + public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack) => func(actions); + + public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack, out IList stdout) + { + stdout = new string[0]; + return func(actions); + } + } + + class BindBuildScript : BuildScript + { + readonly BuildScript s1; + readonly Func, int, BuildScript> s2a; + readonly Func s2b; + public BindBuildScript(BuildScript s1, Func, int, BuildScript> s2) + { + this.s1 = s1; + this.s2a = s2; + } + + public BindBuildScript(BuildScript s1, Func s2) + { + this.s1 = s1; + this.s2b = s2; + } + + public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack) + { + int ret1; + if (s2a != null) + { + ret1 = s1.Run(actions, startCallback, exitCallBack, out var stdout1); + return s2a(stdout1, ret1).Run(actions, startCallback, exitCallBack); + } + + ret1 = s1.Run(actions, startCallback, exitCallBack); + return s2b(ret1).Run(actions, startCallback, exitCallBack); + } + + public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack, out IList stdout) + { + var ret1 = s1.Run(actions, startCallback, exitCallBack, out var stdout1); + var ret2 = (s2a != null ? s2a(stdout1, ret1) : s2b(ret1)).Run(actions, startCallback, exitCallBack, out var stdout2); + var @out = new List(); + @out.AddRange(stdout1); + @out.AddRange(stdout2); + stdout = @out; + return ret2; + } + } + + /// + /// Creates a simple build script that runs the specified exe. + /// + /// The arguments to the executable, or null. + /// The working directory (null for current directory). + /// Additional environment variables. + public static BuildScript Create(string exe, string argumentsOpt, string workingDirectory, IDictionary environment) => + new BuildCommand(exe, argumentsOpt, workingDirectory, environment); + + /// + /// Creates a simple build script that runs the specified function. + /// + public static BuildScript Create(Func func) => + new ReturnBuildCommand(func); + + /// + /// Creates a build script that runs , followed by running the script + /// produced by on the exit code from . + /// + public static BuildScript Bind(BuildScript s1, Func s2) => + new BindBuildScript(s1, s2); + + /// + /// Creates a build script that runs , followed by running the script + /// produced by on the exit code and standard output from + /// . + /// + public static BuildScript Bind(BuildScript s1, Func, int, BuildScript> s2) => + new BindBuildScript(s1, s2); + + const int SuccessCode = 0; + /// + /// The empty build script that always returns exit code 0. + /// + public static readonly BuildScript Success = Create(actions => SuccessCode); + + const int FailureCode = 1; + /// + /// The empty build script that always returns exit code 1. + /// + public static readonly BuildScript Failure = Create(actions => FailureCode); + + static bool Succeeded(int i) => i == SuccessCode; + + public static BuildScript operator &(BuildScript s1, BuildScript s2) => + new BindBuildScript(s1, ret1 => Succeeded(ret1) ? s2 : Create(actions => ret1)); + + public static BuildScript operator &(BuildScript s1, Func s2) => + new BindBuildScript(s1, ret1 => Succeeded(ret1) ? s2() : Create(actions => ret1)); + + public static BuildScript operator |(BuildScript s1, BuildScript s2) => + new BindBuildScript(s1, ret1 => Succeeded(ret1) ? Success : s2); + + public static BuildScript operator |(BuildScript s1, Func s2) => + new BindBuildScript(s1, ret1 => Succeeded(ret1) ? Success : s2()); + + /// + /// Creates a build script that attempts to run the build script , + /// always returning a successful exit code. + /// + public static BuildScript Try(BuildScript s) => s | Success; + + /// + /// Creates a build script that deletes the given directory. + /// + public static BuildScript DeleteDirectory(string dir) => + Create(actions => + { + if (string.IsNullOrEmpty(dir) || !actions.DirectoryExists(dir)) + return FailureCode; + + try + { + actions.DirectoryDelete(dir, true); + } + catch + { + return FailureCode; + } + return SuccessCode; + }); + + /// + /// Creates a build script that deletes the given file. + /// + public static BuildScript DeleteFile(string file) => + Create(actions => + { + if (string.IsNullOrEmpty(file) || !actions.FileExists(file)) + return FailureCode; + + try + { + actions.FileDelete(file); + } + catch + { + return FailureCode; + } + return SuccessCode; + }); + } +} diff --git a/csharp/extractor/Semmle.Autobuild/BuildTools.cs b/csharp/extractor/Semmle.Autobuild/BuildTools.cs new file mode 100644 index 00000000000..ca9285d2c9c --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/BuildTools.cs @@ -0,0 +1,99 @@ +using System.Collections.Generic; +using System.Linq; + +namespace Semmle.Autobuild +{ + /// + /// A BAT file used to initialise the appropriate + /// Visual Studio version/platform. + /// + public class VcVarsBatFile + { + public readonly int ToolsVersion; + public readonly string Path; + public readonly string[] Platform; + + public VcVarsBatFile(string path, int version, params string[] platform) + { + Path = path; + ToolsVersion = version; + Platform = platform; + } + }; + + /// + /// Collection of available Visual Studio build tools. + /// + public static class BuildTools + { + public static IEnumerable GetCandidateVcVarsFiles(IBuildActions actions) + { + var programFilesx86 = actions.GetEnvironmentVariable("ProgramFiles(x86)"); + if (programFilesx86 == null) + yield break; + + // Attempt to use vswhere to find installations of Visual Studio + string vswhere = actions.PathCombine(programFilesx86, "Microsoft Visual Studio", "Installer", "vswhere.exe"); + + if (actions.FileExists(vswhere)) + { + int exitCode1 = actions.RunProcess(vswhere, "-prerelease -legacy -property installationPath", null, null, out var installationList); + int exitCode2 = actions.RunProcess(vswhere, "-prerelease -legacy -property installationVersion", null, null, out var versionList); + + if (exitCode1 == 0 && exitCode2 == 0 && versionList.Count == installationList.Count) + { + // vswhere ran successfully and produced the expected output + foreach (var vsInstallation in versionList.Zip(installationList, (v, i) => (Version: v, InstallationPath: i))) + { + var dot = vsInstallation.Version.IndexOf('.'); + var majorVersionString = dot == -1 ? vsInstallation.Version : vsInstallation.Version.Substring(0, dot); + if (int.TryParse(majorVersionString, out int majorVersion)) + { + if (majorVersion < 15) + { + yield return new VcVarsBatFile(actions.PathCombine(vsInstallation.InstallationPath, @"VC\vcvarsall.bat"), majorVersion, "x86"); + } + else + { + yield return new VcVarsBatFile(actions.PathCombine(vsInstallation.InstallationPath, @"VC\Auxiliary\Build\vcvars32.bat"), majorVersion, "x86"); + yield return new VcVarsBatFile(actions.PathCombine(vsInstallation.InstallationPath, @"VC\Auxiliary\Build\vcvars64.bat"), majorVersion, "x64"); + } + } + // else: Skip installation without a version + } + yield break; + } + } + + // vswhere not installed or didn't run correctly - return legacy Visual Studio versions + yield return new VcVarsBatFile(actions.PathCombine(programFilesx86, @"Microsoft Visual Studio 14.0\VC\vcvarsall.bat"), 14, "x86"); + yield return new VcVarsBatFile(actions.PathCombine(programFilesx86, @"Microsoft Visual Studio 12.0\VC\vcvarsall.bat"), 12, "x86"); + yield return new VcVarsBatFile(actions.PathCombine(programFilesx86, @"Microsoft Visual Studio 11.0\VC\vcvarsall.bat"), 11, "x86"); + yield return new VcVarsBatFile(actions.PathCombine(programFilesx86, @"Microsoft Visual Studio 10.0\VC\vcvarsall.bat"), 10, "x86"); + } + + /// + /// Enumerates all available tools. + /// + public static IEnumerable VcVarsAllBatFiles(IBuildActions actions) => + GetCandidateVcVarsFiles(actions).Where(v => actions.FileExists(v.Path)); + + /// + /// Finds a VcVars file that provides a compatible environment for the given solution. + /// + /// The solution file. + /// A compatible file, or throws an exception. + public static VcVarsBatFile FindCompatibleVcVars(IBuildActions actions, ISolution sln) => + FindCompatibleVcVars(actions, sln.ToolsVersion.Major); + + /// + /// Finds a VcVars that provides a compatible environment for the given tools version. + /// + /// The tools version. + /// A compatible file, or null. + public static VcVarsBatFile FindCompatibleVcVars(IBuildActions actions, int targetVersion) => + targetVersion < 10 ? + VcVarsAllBatFiles(actions).OrderByDescending(b => b.ToolsVersion).FirstOrDefault() : + VcVarsAllBatFiles(actions).Where(b => b.ToolsVersion >= targetVersion).OrderBy(b => b.ToolsVersion).FirstOrDefault(); + } +} diff --git a/csharp/extractor/Semmle.Autobuild/CommandBuilder.cs b/csharp/extractor/Semmle.Autobuild/CommandBuilder.cs new file mode 100644 index 00000000000..273cc52b036 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/CommandBuilder.cs @@ -0,0 +1,195 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Semmle.Autobuild +{ + /// + /// Utility to construct a build command. + /// + class CommandBuilder + { + enum EscapeMode { Process, Cmd }; + + readonly StringBuilder arguments; + bool firstCommand; + string executable; + readonly EscapeMode escapingMode; + readonly string workingDirectory; + readonly IDictionary environment; + + /// + /// Initializes a new instance of the class. + /// + /// The working directory (null for current directory). + /// Additional environment variables. + public CommandBuilder(IBuildActions actions, string workingDirectory = null, IDictionary environment = null) + { + arguments = new StringBuilder(); + if (actions.IsWindows()) + { + executable = "cmd.exe"; + arguments.Append("/C"); + escapingMode = EscapeMode.Cmd; + } + else + { + escapingMode = EscapeMode.Process; + } + + firstCommand = true; + this.workingDirectory = workingDirectory; + this.environment = environment; + } + + void OdasaIndex(string odasa) + { + RunCommand(odasa, "index --auto"); + } + + public CommandBuilder CallBatFile(string batFile, string argumentsOpt = null) + { + NextCommand(); + arguments.Append(" CALL"); + QuoteArgument(batFile); + Argument(argumentsOpt); + return this; + } + + /// + /// Perform odasa index on a given command or BAT file. + /// + /// The odasa executable. + /// The command to run. + /// Additional arguments. + /// this for chaining calls. + public CommandBuilder IndexCommand(string odasa, string command, string argumentsOpt = null) + { + OdasaIndex(odasa); + QuoteArgument(command); + Argument(argumentsOpt); + return this; + } + + static readonly char[] specialChars = { ' ', '\t', '\n', '\v', '\"' }; + static readonly char[] cmdMetacharacter = { '(', ')', '%', '!', '^', '\"', '<', '>', '&', '|' }; + + /// + /// Appends the given argument to the command line. + /// + /// The argument to append. + /// Whether to always quote the argument. + /// Whether to escape for cmd.exe + /// + /// + /// This implementation is copied from + /// https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/ + /// + void ArgvQuote(string argument, bool force) + { + bool cmd = escapingMode == EscapeMode.Cmd; + if (!force && + !string.IsNullOrEmpty(argument) && + argument.IndexOfAny(specialChars) == -1) + { + arguments.Append(argument); + } + else + { + if (cmd) arguments.Append('^'); + arguments.Append('\"'); + for (int it = 0; ; ++it) + { + var numBackslashes = 0; + while (it != argument.Length && argument[it] == '\\') + { + ++it; + ++numBackslashes; + } + + if (it == argument.Length) + { + arguments.Append('\\', numBackslashes * 2); + break; + } + else if (argument[it] == '\"') + { + arguments.Append('\\', numBackslashes * 2 + 1); + if (cmd) arguments.Append('^'); + arguments.Append(arguments[it]); + } + else + { + arguments.Append('\\', numBackslashes); + if (cmd && cmdMetacharacter.Any(c => c == argument[it])) + arguments.Append('^'); + + arguments.Append(argument[it]); + } + } + if (cmd) arguments.Append('^'); + arguments.Append('\"'); + } + } + + public CommandBuilder QuoteArgument(string argumentsOpt) + { + if (argumentsOpt != null) + { + NextArgument(); + ArgvQuote(argumentsOpt, false); + } + return this; + } + + void NextArgument() + { + if (arguments.Length > 0) + arguments.Append(' '); + } + + public CommandBuilder Argument(string argumentsOpt) + { + if (argumentsOpt != null) + { + NextArgument(); + arguments.Append(argumentsOpt); + } + return this; + } + + void NextCommand() + { + if (firstCommand) + firstCommand = false; + else + arguments.Append(" &&"); + } + + public CommandBuilder RunCommand(string exe, string argumentsOpt = null) + { + var (exe0, arg0) = + escapingMode == EscapeMode.Process && exe.EndsWith(".exe", System.StringComparison.Ordinal) + ? ("mono", exe) // Linux + : (exe, null); + + NextCommand(); + if (executable == null) + { + executable = exe0; + } + else + { + QuoteArgument(exe0); + } + Argument(arg0); + Argument(argumentsOpt); + return this; + } + + /// + /// Returns a build script that contains just this command. + /// + public BuildScript Script => BuildScript.Create(executable, arguments.ToString(), workingDirectory, environment); + } +} diff --git a/csharp/extractor/Semmle.Autobuild/DotNetRule.cs b/csharp/extractor/Semmle.Autobuild/DotNetRule.cs new file mode 100644 index 00000000000..a862a4f1a1b --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/DotNetRule.cs @@ -0,0 +1,268 @@ +using System; +using Semmle.Util.Logging; +using System.Linq; +using Newtonsoft.Json.Linq; +using System.Collections.Generic; +using System.IO; + +namespace Semmle.Autobuild +{ + /// + /// A build rule where the build command is of the form "dotnet build". + /// Currently unused because the tracer does not work with dotnet. + /// + class DotNetRule : IBuildRule + { + public BuildScript Analyse(Autobuilder builder) + { + builder.Log(Severity.Info, "Attempting to build using .NET Core"); + + var projects = builder.SolutionsToBuild.Any() + ? builder.SolutionsToBuild.SelectMany(s => s.Projects).ToArray() + : builder.GetExtensions(Language.CSharp.ProjectExtension).Select(p => new Project(builder, p)).ToArray(); + + var notDotNetProject = projects.FirstOrDefault(p => !p.DotNetProject); + if (notDotNetProject != null) + { + builder.Log(Severity.Info, "Not using .NET Core because of incompatible project {0}", notDotNetProject); + return BuildScript.Failure; + } + + if (!builder.SolutionsToBuild.Any()) + // Attempt dotnet build in root folder + return WithDotNet(builder, dotNet => + { + var info = GetInfoCommand(builder.Actions, dotNet); + var clean = GetCleanCommand(builder.Actions, dotNet).Script; + var restore = GetRestoreCommand(builder.Actions, dotNet).Script; + var build = GetBuildCommand(builder, dotNet).Script; + return info & clean & BuildScript.Try(restore) & build; + }); + + // Attempt dotnet build on each solution + return WithDotNet(builder, dotNet => + { + var ret = GetInfoCommand(builder.Actions, dotNet); + foreach (var solution in builder.SolutionsToBuild) + { + var cleanCommand = GetCleanCommand(builder.Actions, dotNet); + cleanCommand.QuoteArgument(solution.Path); + var clean = cleanCommand.Script; + + var restoreCommand = GetRestoreCommand(builder.Actions, dotNet); + restoreCommand.QuoteArgument(solution.Path); + var restore = restoreCommand.Script; + + var buildCommand = GetBuildCommand(builder, dotNet); + buildCommand.QuoteArgument(solution.Path); + var build = buildCommand.Script; + + ret &= clean & BuildScript.Try(restore) & build; + } + return ret; + }); + } + + /// + /// Returns a script that attempts to download relevant version(s) of the + /// .NET Core SDK, followed by running the script generated by . + /// + /// The first element DotNetPath of the argument to + /// is the path where .NET Core was installed, and the second element Environment + /// is any additional required environment variables. The tuple argument is null + /// when the installation failed. + /// + public static BuildScript WithDotNet(Autobuilder builder, Func<(string DotNetPath, IDictionary Environment)?, BuildScript> f) + { + var installDir = builder.Actions.PathCombine(builder.Options.RootDirectory, ".dotnet"); + var installScript = DownloadDotNet(builder, installDir); + return BuildScript.Bind(installScript, installed => + { + if (installed == 0) + { + // The installation succeeded, so use the newly installed .NET Core + var path = builder.Actions.GetEnvironmentVariable("PATH"); + var delim = builder.Actions.IsWindows() ? ";" : ":"; + var env = new Dictionary{ + { "DOTNET_MULTILEVEL_LOOKUP", "false" }, // prevent look up of other .NET Core SDKs + { "DOTNET_SKIP_FIRST_TIME_EXPERIENCE", "true" }, + { "PATH", installDir + delim + path } + }; + return f((installDir, env)); + } + + return f(null); + }); + } + + /// + /// Returns a script for downloading relevant versions of the + /// .NET Core SDK. The SDK(s) will be installed at installDir + /// (provided that the script succeeds). + /// + static BuildScript DownloadDotNet(Autobuilder builder, string installDir) + { + if (!string.IsNullOrEmpty(builder.Options.DotNetVersion)) + // Specific version supplied in configuration: always use that + return DownloadDotNetVersion(builder, installDir, builder.Options.DotNetVersion); + + // Download versions mentioned in `global.json` files + // See https://docs.microsoft.com/en-us/dotnet/core/tools/global-json + var installScript = BuildScript.Success; + var validGlobalJson = false; + foreach (var path in builder.Paths.Where(p => p.EndsWith("global.json", StringComparison.Ordinal))) + { + string version; + try + { + var o = JObject.Parse(File.ReadAllText(path)); + version = (string)o["sdk"]["version"]; + } + catch + { + // not a valid global.json file + continue; + } + + installScript &= DownloadDotNetVersion(builder, installDir, version); + validGlobalJson = true; + } + + return validGlobalJson ? installScript : BuildScript.Failure; + } + + /// + /// Returns a script for downloading a specific .NET Core SDK version, if the + /// version is not already installed. + /// + /// See https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script. + /// + static BuildScript DownloadDotNetVersion(Autobuilder builder, string path, string version) + { + return BuildScript.Bind(GetInstalledSdksScript(builder.Actions), (sdks, sdksRet) => + { + if (sdksRet == 0 && sdks.Count() == 1 && sdks[0].StartsWith(version + " ", StringComparison.Ordinal)) + // The requested SDK is already installed (and no other SDKs are installed), so + // no need to reinstall + return BuildScript.Failure; + + builder.Log(Severity.Info, "Attempting to download .NET Core {0}", version); + + if (builder.Actions.IsWindows()) + { + var psScript = @"param([string]$Version, [string]$InstallDir) + +add-type @"" +using System.Net; +using System.Security.Cryptography.X509Certificates; +public class TrustAllCertsPolicy : ICertificatePolicy +{ + public bool CheckValidationResult(ServicePoint srvPoint, X509Certificate certificate, WebRequest request, int certificateProblem) + { + return true; + } +} +""@ +$AllProtocols = [System.Net.SecurityProtocolType]'Ssl3,Tls,Tls11,Tls12' +[System.Net.ServicePointManager]::SecurityProtocol = $AllProtocols +[System.Net.ServicePointManager]::CertificatePolicy = New-Object TrustAllCertsPolicy +$Script = Invoke-WebRequest -useb 'https://dot.net/v1/dotnet-install.ps1' + +$arguments = @{ + Channel = 'release' + Version = $Version + InstallDir = $InstallDir +} + +$ScriptBlock = [scriptblock]::create("".{$($Script)} $(&{$args} @arguments)"") + +Invoke-Command -ScriptBlock $ScriptBlock"; + var psScriptFile = builder.Actions.PathCombine(builder.Options.RootDirectory, "install-dotnet.ps1"); + builder.Actions.WriteAllText(psScriptFile, psScript); + + var install = new CommandBuilder(builder.Actions). + RunCommand("powershell"). + Argument("-NoProfile"). + Argument("-ExecutionPolicy"). + Argument("unrestricted"). + Argument("-file"). + Argument(psScriptFile). + Argument("-Version"). + Argument(version). + Argument("-InstallDir"). + Argument(path); + return install.Script; + } + else + { + var curl = new CommandBuilder(builder.Actions). + RunCommand("curl"). + Argument("-sO"). + Argument("https://dot.net/v1/dotnet-install.sh"); + + var chmod = new CommandBuilder(builder.Actions). + RunCommand("chmod"). + Argument("u+x"). + Argument("dotnet-install.sh"); + + var install = new CommandBuilder(builder.Actions). + RunCommand("./dotnet-install.sh"). + Argument("--channel"). + Argument("release"). + Argument("--version"). + Argument(version). + Argument("--install-dir"). + Argument(path); + + return curl.Script & chmod.Script & install.Script; + } + }); + } + + static BuildScript GetInstalledSdksScript(IBuildActions actions) + { + var listSdks = new CommandBuilder(actions). + RunCommand("dotnet"). + Argument("--list-sdks"); + return listSdks.Script; + } + + static string DotNetCommand(IBuildActions actions, string dotNetPath) => + dotNetPath != null ? actions.PathCombine(dotNetPath, "dotnet") : "dotnet"; + + BuildScript GetInfoCommand(IBuildActions actions, (string DotNetPath, IDictionary Environment)? arg) + { + var info = new CommandBuilder(actions, null, arg?.Environment). + RunCommand(DotNetCommand(actions, arg?.DotNetPath)). + Argument("--info"); + return info.Script; + } + + CommandBuilder GetCleanCommand(IBuildActions actions, (string DotNetPath, IDictionary Environment)? arg) + { + var clean = new CommandBuilder(actions, null, arg?.Environment). + RunCommand(DotNetCommand(actions, arg?.DotNetPath)). + Argument("clean"); + return clean; + } + + CommandBuilder GetRestoreCommand(IBuildActions actions, (string DotNetPath, IDictionary Environment)? arg) + { + var restore = new CommandBuilder(actions, null, arg?.Environment). + RunCommand(DotNetCommand(actions, arg?.DotNetPath)). + Argument("restore"); + return restore; + } + + CommandBuilder GetBuildCommand(Autobuilder builder, (string DotNetPath, IDictionary Environment)? arg) + { + var build = new CommandBuilder(builder.Actions, null, arg?.Environment). + IndexCommand(builder.Odasa, DotNetCommand(builder.Actions, arg?.DotNetPath)). + Argument("build"). + Argument("--no-incremental"). + Argument("/p:UseSharedCompilation=false"). + Argument(builder.Options.DotNetArguments); + return build; + } + } +} diff --git a/csharp/extractor/Semmle.Autobuild/Language.cs b/csharp/extractor/Semmle.Autobuild/Language.cs new file mode 100644 index 00000000000..6f8d73b2a47 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/Language.cs @@ -0,0 +1,24 @@ +namespace Semmle.Autobuild +{ + public sealed class Language + { + public static readonly Language Cpp = new Language(".vcxproj"); + public static readonly Language CSharp = new Language(".csproj"); + + public bool ProjectFileHasThisLanguage(string path) => + System.IO.Path.GetExtension(path) == ProjectExtension; + + public static bool IsProjectFileForAnySupportedLanguage(string path) => + Cpp.ProjectFileHasThisLanguage(path) || CSharp.ProjectFileHasThisLanguage(path); + + public readonly string ProjectExtension; + + private Language(string extension) + { + ProjectExtension = extension; + } + + public override string ToString() => + ProjectExtension == Cpp.ProjectExtension ? "C/C++" : "C#"; + } +} diff --git a/csharp/extractor/Semmle.Autobuild/MsBuildRule.cs b/csharp/extractor/Semmle.Autobuild/MsBuildRule.cs new file mode 100644 index 00000000000..cb7eae2c23b --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/MsBuildRule.cs @@ -0,0 +1,118 @@ +using Semmle.Util.Logging; +using System.IO; +using System.Linq; + +namespace Semmle.Autobuild +{ + /// + /// A build rule using msbuild. + /// + class MsBuildRule : IBuildRule + { + /// + /// The name of the msbuild command. + /// + const string MsBuild = "msbuild"; + + public BuildScript Analyse(Autobuilder builder) + { + builder.Log(Severity.Info, "Attempting to build using MSBuild"); + + if (!builder.SolutionsToBuild.Any()) + { + builder.Log(Severity.Info, "Could not find a suitable solution file to build"); + return BuildScript.Failure; + } + + var vsTools = GetVcVarsBatFile(builder); + + if (vsTools == null && builder.SolutionsToBuild.Any()) + { + vsTools = BuildTools.FindCompatibleVcVars(builder.Actions, builder.SolutionsToBuild.First()); + } + + if (vsTools == null && builder.Actions.IsWindows()) + { + builder.Log(Severity.Warning, "Could not find a suitable version of vcvarsall.bat"); + } + + var nuget = builder.Actions.PathCombine(builder.SemmlePlatformTools, "csharp", "nuget", "nuget.exe"); + + var ret = BuildScript.Success; + + foreach (var solution in builder.SolutionsToBuild) + { + if (builder.Options.NugetRestore) + { + var nugetCommand = new CommandBuilder(builder.Actions). + RunCommand(nuget). + Argument("restore"). + QuoteArgument(solution.Path); + ret &= BuildScript.Try(nugetCommand.Script); + } + + var command = new CommandBuilder(builder.Actions); + + if (vsTools != null) + { + command.CallBatFile(vsTools.Path); + } + + command.IndexCommand(builder.Odasa, MsBuild); + command.QuoteArgument(solution.Path); + + command.Argument("/p:UseSharedCompilation=false"); + + string target = builder.Options.MsBuildTarget != null ? builder.Options.MsBuildTarget : "rebuild"; + string platform = builder.Options.MsBuildPlatform != null ? builder.Options.MsBuildPlatform : solution.DefaultPlatformName; + string configuration = builder.Options.MsBuildConfiguration != null ? builder.Options.MsBuildConfiguration : solution.DefaultConfigurationName; + + command.Argument("/t:" + target); + command.Argument(string.Format("/p:Platform=\"{0}\"", platform)); + command.Argument(string.Format("/p:Configuration=\"{0}\"", configuration)); + command.Argument("/p:MvcBuildViews=true"); + + command.Argument(builder.Options.MsBuildArguments); + + ret &= command.Script; + } + + return ret; + } + + /// + /// Gets the BAT file used to initialize the appropriate Visual Studio + /// version/platform, as specified by the `vstools_version` property in + /// lgtm.yml. + /// + /// Returns null when no version is specified. + /// + public static VcVarsBatFile GetVcVarsBatFile(Autobuilder builder) + { + VcVarsBatFile vsTools = null; + + if (builder.Options.VsToolsVersion != null) + { + if (int.TryParse(builder.Options.VsToolsVersion, out var msToolsVersion)) + { + foreach (var b in BuildTools.VcVarsAllBatFiles(builder.Actions)) + { + builder.Log(Severity.Info, "Found {0} version {1}", b.Path, b.ToolsVersion); + } + + vsTools = BuildTools.FindCompatibleVcVars(builder.Actions, msToolsVersion); + if (vsTools == null) + builder.Log(Severity.Warning, "Could not find build tools matching version {0}", msToolsVersion); + else + builder.Log(Severity.Info, "Setting Visual Studio tools to {0}", vsTools.Path); + } + else + { + builder.Log(Severity.Error, "The format of vstools_version is incorrect. Please specify an integer."); + } + } + + return vsTools; + } + } +} diff --git a/csharp/extractor/Semmle.Autobuild/Program.cs b/csharp/extractor/Semmle.Autobuild/Program.cs new file mode 100644 index 00000000000..c4542864a09 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/Program.cs @@ -0,0 +1,28 @@ +using System; + +namespace Semmle.Autobuild +{ + class Program + { + static int Main() + { + var options = new AutobuildOptions(); + var actions = SystemBuildActions.Instance; + + try + { + options.ReadEnvironment(actions); + } + catch (ArgumentOutOfRangeException ex) + { + Console.WriteLine("The value \"{0}\" for parameter \"{1}\" is invalid", ex.ActualValue, ex.ParamName); + } + + var builder = new Autobuilder(actions, options); + + Console.WriteLine($"Semmle autobuilder for {options.Language}"); + + return builder.AttemptBuild(); + } + } +} diff --git a/csharp/extractor/Semmle.Autobuild/Project.cs b/csharp/extractor/Semmle.Autobuild/Project.cs new file mode 100644 index 00000000000..e0a1ee8386d --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/Project.cs @@ -0,0 +1,69 @@ +using System; +using System.IO; +using System.Xml; +using Semmle.Util.Logging; + +namespace Semmle.Autobuild +{ + /// + /// Representation of a .csproj (C#) or .vcxproj (C++) file. + /// C# project files come in 2 flavours, .Net core and msbuild, but they + /// have the same file extension. + /// + public class Project + { + /// + /// Holds if this project is for .Net core. + /// + public bool DotNetProject { get; private set; } + + public bool ValidToolsVersion { get; private set; } + + public Version ToolsVersion { get; private set; } + + readonly string filename; + + public Project(Autobuilder builder, string filename) + { + this.filename = filename; + ToolsVersion = new Version(); + + if (!File.Exists(filename)) + return; + + var projFile = new XmlDocument(); + projFile.Load(filename); + var root = projFile.DocumentElement; + + if (root.Name == "Project") + { + if (root.HasAttribute("Sdk")) + { + DotNetProject = true; + } + else + { + var toolsVersion = root.GetAttribute("ToolsVersion"); + if (string.IsNullOrEmpty(toolsVersion)) + { + builder.Log(Severity.Warning, "Project {0} is missing a tools version", filename); + } + else + { + try + { + ToolsVersion = new Version(toolsVersion); + ValidToolsVersion = true; + } + catch // Generic catch clause - Version constructor throws about 5 different exceptions. + { + builder.Log(Severity.Warning, "Project {0} has invalid tools version {1}", filename, toolsVersion); + } + } + } + } + } + + public override string ToString() => filename; + } +} diff --git a/csharp/extractor/Semmle.Autobuild/Properties/AssemblyInfo.cs b/csharp/extractor/Semmle.Autobuild/Properties/AssemblyInfo.cs new file mode 100644 index 00000000000..e3da7ca22e9 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/Properties/AssemblyInfo.cs @@ -0,0 +1,35 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Semmle.Autobuild")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("Semmle")] +[assembly: AssemblyProduct("Semmle Visual Studio Autobuild")] +[assembly: AssemblyCopyright("Copyright © Semmle 2017")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("1d9920ad-7b00-4df1-8b01-9ff5b687828e")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/csharp/extractor/Semmle.Autobuild/Semmle.Autobuild.csproj b/csharp/extractor/Semmle.Autobuild/Semmle.Autobuild.csproj new file mode 100644 index 00000000000..8a9d272169c --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/Semmle.Autobuild.csproj @@ -0,0 +1,27 @@ + + + + netcoreapp2.0 + Semmle.Autobuild + Semmle.Autobuild + + Exe + + false + + + + + + + + + + + + + + + + + diff --git a/csharp/extractor/Semmle.Autobuild/Solution.cs b/csharp/extractor/Semmle.Autobuild/Solution.cs new file mode 100644 index 00000000000..66bbff777ac --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/Solution.cs @@ -0,0 +1,107 @@ +using Microsoft.Build.Construction; +using Microsoft.Build.Exceptions; +using System; +using System.Collections.Generic; +using System.Linq; +using Semmle.Util; + +namespace Semmle.Autobuild +{ + /// + /// A solution file, extension .sln. + /// + public interface ISolution + { + /// + /// List of C# or C++ projects contained in the solution. + /// (There could be other project types as well - these are ignored.) + /// + + IEnumerable Projects { get; } + + /// + /// Solution configurations. + /// + IEnumerable Configurations { get; } + + /// + /// The default configuration name, e.g. "Release" + /// + string DefaultConfigurationName { get; } + + /// + /// The default platform name, e.g. "x86" + /// + string DefaultPlatformName { get; } + + /// + /// The path of the solution file. + /// + string Path { get; } + + /// + /// The number of C# or C++ projects. + /// + int ProjectCount { get; } + + /// + /// Gets the "best" tools version for this solution. + /// If there are several versions, because the project files + /// are inconsistent, then pick the highest/latest version. + /// If no tools versions are present, return 0.0.0.0. + /// + Version ToolsVersion { get; } + } + + /// + /// A solution file on the filesystem, read using Microsoft.Build. + /// + class Solution : ISolution + { + readonly SolutionFile solution; + + public IEnumerable Projects { get; private set; } + + public IEnumerable Configurations => + solution == null ? Enumerable.Empty() : solution.SolutionConfigurations; + + public string DefaultConfigurationName => + solution == null ? "" : solution.GetDefaultConfigurationName(); + + public string DefaultPlatformName => + solution == null ? "" : solution.GetDefaultPlatformName(); + + public Solution(Autobuilder builder, string path) + { + Path = System.IO.Path.GetFullPath(path); + try + { + solution = SolutionFile.Parse(Path); + + Projects = + solution.ProjectsInOrder. + Where(p => p.ProjectType == SolutionProjectType.KnownToBeMSBuildFormat). + Select(p => System.IO.Path.GetFullPath(FileUtils.ConvertToNative(p.AbsolutePath))). + Where(p => builder.Options.Language.ProjectFileHasThisLanguage(p)). + Select(p => new Project(builder, p)). + ToArray(); + } + catch (InvalidProjectFileException) + { + // We allow specifying projects as solutions in lgtm.yml, so model + // that scenario as a solution with just that one project + Projects = Language.IsProjectFileForAnySupportedLanguage(Path) + ? new[] { new Project(builder, Path) } + : new Project[0]; + } + } + + public string Path { get; private set; } + + public int ProjectCount => Projects.Count(); + + IEnumerable ToolsVersions => Projects.Where(p => p.ValidToolsVersion).Select(p => p.ToolsVersion); + + public Version ToolsVersion => ToolsVersions.Any() ? ToolsVersions.Max() : new Version(); + } +} diff --git a/csharp/extractor/Semmle.Autobuild/StandaloneBuildRule.cs b/csharp/extractor/Semmle.Autobuild/StandaloneBuildRule.cs new file mode 100644 index 00000000000..3fafa84e454 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/StandaloneBuildRule.cs @@ -0,0 +1,46 @@ +using System.IO; + +namespace Semmle.Autobuild +{ + /// + /// Build using standalone extraction. + /// + class StandaloneBuildRule : IBuildRule + { + public BuildScript Analyse(Autobuilder builder) + { + BuildScript GetCommand(string solution) + { + var standalone = builder.Actions.PathCombine(builder.SemmlePlatformTools, "csharp", "Semmle.Extraction.CSharp.Standalone"); + var cmd = new CommandBuilder(builder.Actions); + cmd.RunCommand(standalone); + + if (solution != null) + cmd.QuoteArgument(solution); + + cmd.Argument("--references:."); + + if (!builder.Options.NugetRestore) + { + cmd.Argument("--skip-nuget"); + } + + return cmd.Script; + } + + if (!builder.Options.Buildless) + return BuildScript.Failure; + + var solutions = builder.Options.Solution.Length; + + if (solutions == 0) + return GetCommand(null); + + var script = BuildScript.Success; + foreach (var solution in builder.Options.Solution) + script &= GetCommand(solution); + + return script; + } + } +} diff --git a/csharp/extractor/Semmle.Autobuild/XmlBuildRule.cs b/csharp/extractor/Semmle.Autobuild/XmlBuildRule.cs new file mode 100644 index 00000000000..6b0a5b1cbb1 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/XmlBuildRule.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; + +namespace Semmle.Autobuild +{ + /// + /// XML extraction. + /// + class XmlBuildRule : IBuildRule + { + public BuildScript Analyse(Autobuilder builder) + { + var command = new CommandBuilder(builder.Actions). + RunCommand(builder.Odasa). + Argument("index --xml --extensions config"); + return command.Script; + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL.Driver/ExtractorOptions.cs b/csharp/extractor/Semmle.Extraction.CIL.Driver/ExtractorOptions.cs new file mode 100644 index 00000000000..0e9a8f9e116 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL.Driver/ExtractorOptions.cs @@ -0,0 +1,267 @@ +using System.Collections.Generic; +using System.Linq; +using System.IO; +using System.Reflection.PortableExecutable; +using System.Reflection.Metadata; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Globalization; +using Semmle.Util.Logging; + +namespace Semmle.Extraction.CIL.Driver +{ + /// + /// Information about a single assembly. + /// In particular, provides references between assemblies. + /// + class AssemblyInfo + { + public override string ToString() => filename; + + static AssemblyName CreateAssemblyName(MetadataReader mdReader, StringHandle name, System.Version version, StringHandle culture) + { + var cultureString = mdReader.GetString(culture); + + var assemblyName = new AssemblyName() + { + Name = mdReader.GetString(name), + Version = version + }; + + if (cultureString != "neutral") + assemblyName.CultureInfo = CultureInfo.GetCultureInfo(cultureString); + + return assemblyName; + } + + static AssemblyName CreateAssemblyName(MetadataReader mdReader, AssemblyReference ar) + { + var an = CreateAssemblyName(mdReader, ar.Name, ar.Version, ar.Culture); + if (!ar.PublicKeyOrToken.IsNil) + an.SetPublicKeyToken(mdReader.GetBlobBytes(ar.PublicKeyOrToken)); + return an; + } + + static AssemblyName CreateAssemblyName(MetadataReader mdReader, AssemblyDefinition ad) + { + var an = CreateAssemblyName(mdReader, ad.Name, ad.Version, ad.Culture); + if (!ad.PublicKey.IsNil) + an.SetPublicKey(mdReader.GetBlobBytes(ad.PublicKey)); + return an; + } + + public AssemblyInfo(string path) + { + filename = path; + + // Attempt to open the file and see if it's a valid assembly. + using (var stream = File.OpenRead(path)) + using (var peReader = new PEReader(stream)) + { + try + { + isAssembly = peReader.HasMetadata; + if (!isAssembly) return; + + var mdReader = peReader.GetMetadataReader(); + + isAssembly = mdReader.IsAssembly; + if (!mdReader.IsAssembly) return; + + // Get our own assembly name + name = CreateAssemblyName(mdReader, mdReader.GetAssemblyDefinition()); + + references = mdReader.AssemblyReferences. + Select(r => mdReader.GetAssemblyReference(r)). + Select(ar => CreateAssemblyName(mdReader, ar)). + ToArray(); + } + catch (System.BadImageFormatException) + { + // This failed on one of the Roslyn tests that includes + // a deliberately malformed assembly. + // In this case, we just skip the extraction of this assembly. + isAssembly = false; + } + } + } + + public readonly AssemblyName name; + public readonly string filename; + public bool extract; + public readonly bool isAssembly; + public readonly AssemblyName[] references; + } + + /// + /// Helper to manage a collection of assemblies. + /// Resolves references between assemblies and determines which + /// additional assemblies need to be extracted. + /// + class AssemblyList + { + class AssemblyNameComparer : IEqualityComparer + { + bool IEqualityComparer.Equals(AssemblyName x, AssemblyName y) => + x.Name == y.Name && x.Version == y.Version; + + int IEqualityComparer.GetHashCode(AssemblyName obj) => + obj.Name.GetHashCode() + 7 * obj.Version.GetHashCode(); + } + + readonly Dictionary assembliesRead = new Dictionary(new AssemblyNameComparer()); + + public void AddFile(string assemblyPath, bool extractAll) + { + if (!filesAnalyzed.Contains(assemblyPath)) + { + filesAnalyzed.Add(assemblyPath); + var info = new AssemblyInfo(assemblyPath); + if (info.isAssembly) + { + info.extract = extractAll; + if (!assembliesRead.ContainsKey(info.name)) + assembliesRead.Add(info.name, info); + } + } + } + + public IEnumerable AssembliesToExtract => assembliesRead.Values.Where(info => info.extract); + + IEnumerable AssembliesToReference => AssembliesToExtract.SelectMany(info => info.references); + + public void ResolveReferences() + { + var assembliesToReference = new Stack(AssembliesToReference); + + while (assembliesToReference.Any()) + { + var item = assembliesToReference.Pop(); + AssemblyInfo info; + if (assembliesRead.TryGetValue(item, out info)) + { + if (!info.extract) + { + info.extract = true; + foreach (var reference in info.references) + assembliesToReference.Push(reference); + } + } + else + { + missingReferences.Add(item); + } + } + } + + readonly HashSet filesAnalyzed = new HashSet(); + public readonly HashSet missingReferences = new HashSet(); + } + + /// + /// Parses the command line and collates a list of DLLs/EXEs to extract. + /// + class ExtractorOptions + { + readonly AssemblyList assemblyList = new AssemblyList(); + + public void AddDirectory(string directory, bool extractAll) + { + foreach (var file in + Directory.EnumerateFiles(directory, "*.dll", SearchOption.AllDirectories). + Concat(Directory.EnumerateFiles(directory, "*.exe", SearchOption.AllDirectories))) + { + assemblyList.AddFile(file, extractAll); + } + } + + void AddFrameworkDirectories(bool extractAll) + { + AddDirectory(RuntimeEnvironment.GetRuntimeDirectory(), extractAll); + } + + public Verbosity Verbosity { get; private set; } + public bool NoCache { get; private set; } + public int Threads { get; private set; } + public bool PDB { get; private set; } + + void AddFileOrDirectory(string path) + { + path = Path.GetFullPath(path); + if (File.Exists(path)) + { + assemblyList.AddFile(path, true); + AddDirectory(Path.GetDirectoryName(path), false); + } + else if (Directory.Exists(path)) + { + AddDirectory(path, true); + } + } + + void ResolveReferences() + { + assemblyList.ResolveReferences(); + AssembliesToExtract = assemblyList.AssembliesToExtract.ToArray(); + } + + public IEnumerable AssembliesToExtract { get; private set; } + + /// + /// Gets the assemblies that were referenced but were not available to be + /// extracted. This is not an error, it just means that the database is not + /// as complete as it could be. + /// + public IEnumerable MissingReferences => assemblyList.missingReferences; + + public static ExtractorOptions ParseCommandLine(string[] args) + { + var options = new ExtractorOptions(); + options.Verbosity = Verbosity.Info; + options.Threads = System.Environment.ProcessorCount; + options.PDB = true; + + foreach (var arg in args) + { + if (arg == "--verbose") + { + options.Verbosity = Verbosity.All; + } + else if (arg == "--silent") + { + options.Verbosity = Verbosity.Off; + } + else if (arg.StartsWith("--verbosity:")) + { + options.Verbosity = (Verbosity)int.Parse(arg.Substring(12)); + } + else if (arg == "--dotnet") + { + options.AddFrameworkDirectories(true); + } + else if (arg == "--nocache") + { + options.NoCache = true; + } + else if (arg.StartsWith("--threads:")) + { + options.Threads = int.Parse(arg.Substring(10)); + } + else if (arg == "--no-pdb") + { + options.PDB = false; + } + else + { + options.AddFileOrDirectory(arg); + } + } + + options.AddFrameworkDirectories(false); + options.ResolveReferences(); + + return options; + } + + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs b/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs new file mode 100644 index 00000000000..df17f5a187e --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs @@ -0,0 +1,68 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using System.IO; +using Semmle.Util.Logging; +using System.Diagnostics; + +namespace Semmle.Extraction.CIL.Driver +{ + class Program + { + static void DisplayHelp() + { + Console.WriteLine("CIL command line extractor"); + Console.WriteLine(); + Console.WriteLine("Usage: Semmle.Extraction.CIL.Driver.exe [options] path ..."); + Console.WriteLine(" --verbose Turn on verbose output"); + Console.WriteLine(" --dotnet Extract the .Net Framework"); + Console.WriteLine(" --nocache Overwrite existing trap files"); + Console.WriteLine(" --no-pdb Do not extract PDB files"); + Console.WriteLine(" path A directory/dll/exe to analyze"); + } + + static void ExtractAssembly(Layout layout, string assemblyPath, ILogger logger, bool nocache, bool extractPdbs) + { + string trapFile; + bool extracted; + var sw = new Stopwatch(); + sw.Start(); + Entities.Assembly.ExtractCIL(layout, assemblyPath, logger, nocache, extractPdbs, out trapFile, out extracted); + sw.Stop(); + logger.Log(Severity.Info, " {0} ({1})", assemblyPath, sw.Elapsed); + } + + static void Main(string[] args) + { + if (args.Length == 0) + { + DisplayHelp(); + return; + } + + var options = ExtractorOptions.ParseCommandLine(args); + var layout = new Layout(); + var logger = new ConsoleLogger(options.Verbosity); + + var actions = options. + AssembliesToExtract.Select(asm => asm.filename). + Select(filename => () => ExtractAssembly(layout, filename, logger, options.NoCache, options.PDB)). + ToArray(); + + foreach (var missingRef in options.MissingReferences) + logger.Log(Severity.Info, " Missing assembly " + missingRef); + + var sw = new Stopwatch(); + sw.Start(); + var piOptions = new ParallelOptions + { + MaxDegreeOfParallelism = options.Threads + }; + + Parallel.Invoke(piOptions, actions); + + sw.Stop(); + logger.Log(Severity.Info, "Extraction completed in {0}", sw.Elapsed); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL.Driver/Properties/AssemblyInfo.cs b/csharp/extractor/Semmle.Extraction.CIL.Driver/Properties/AssemblyInfo.cs new file mode 100644 index 00000000000..56f7f94c143 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL.Driver/Properties/AssemblyInfo.cs @@ -0,0 +1,35 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Semmle.Extraction.CIL.Driver")] +[assembly: AssemblyDescription("Semmle CIL extractor")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("Semmle Ltd")] +[assembly: AssemblyProduct("Semmle.Extraction.CIL.Driver")] +[assembly: AssemblyCopyright("Copyright © Semmle 2017")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("5642ae68-9c26-43c9-bd3c-49923dddf02d")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/csharp/extractor/Semmle.Extraction.CIL.Driver/Semmle.Extraction.CIL.Driver.csproj b/csharp/extractor/Semmle.Extraction.CIL.Driver/Semmle.Extraction.CIL.Driver.csproj new file mode 100644 index 00000000000..b0e1a525490 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL.Driver/Semmle.Extraction.CIL.Driver.csproj @@ -0,0 +1,21 @@ + + + + Exe + netcoreapp2.0 + Semmle.Extraction.CIL.Driver + Semmle.Extraction.CIL.Driver + false + + + + + + + + + + + + + diff --git a/csharp/extractor/Semmle.Extraction.CIL/CachedFunction.cs b/csharp/extractor/Semmle.Extraction.CIL/CachedFunction.cs new file mode 100644 index 00000000000..3bbc386a691 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/CachedFunction.cs @@ -0,0 +1,69 @@ +using System; +using System.Collections.Generic; + +namespace Semmle.Extraction.CIL +{ + /// + /// A factory and a cache for mapping source entities to target entities. + /// Could be considered as a memoizer. + /// + /// The type of the source. + /// The type of the generated object. + public class CachedFunction + { + readonly Func generator; + readonly Dictionary cache; + + /// + /// Initializes the factory with a given mapping. + /// + /// The mapping. + public CachedFunction(Func g) + { + generator = g; + cache = new Dictionary(); + } + + /// + /// Gets the target for a given source. + /// Create it if it does not exist. + /// + /// The source object. + /// The created object. + public TargetType this[SrcType src] + { + get + { + TargetType result; + if (!cache.TryGetValue(src, out result)) + { + result = generator(src); + cache[src] = result; + } + return result; + } + } + } + + /// + /// A factory for mapping a pair of source entities to a target entity. + /// + /// Source entity type 1. + /// Source entity type 2. + /// The target type. + public class CachedFunction + { + readonly CachedFunction<(Src1, Src2), Target> factory; + + /// + /// Initializes the factory with a given mapping. + /// + /// The mapping. + public CachedFunction(Func g) + { + factory = new CachedFunction<(Src1, Src2), Target>(p => g(p.Item1, p.Item2)); + } + + public Target this[Src1 s1, Src2 s2] => factory[(s1, s2)]; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Context.cs b/csharp/extractor/Semmle.Extraction.CIL/Context.cs new file mode 100644 index 00000000000..dc2801e5bc7 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Context.cs @@ -0,0 +1,173 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; + +namespace Semmle.Extraction.CIL +{ + /// + /// Extraction context for CIL extraction. + /// Adds additional context that is specific for CIL extraction. + /// One context = one DLL/EXE. + /// + partial class Context : IDisposable + { + public Extraction.Context cx; + readonly FileStream stream; + public readonly MetadataReader mdReader; + public readonly PEReader peReader; + public readonly string assemblyPath; + public Entities.Assembly assembly; + public PDB.IPdb pdb; + + public Context(Extraction.Context cx, string assemblyPath, bool extractPdbs) + { + this.cx = cx; + this.assemblyPath = assemblyPath; + stream = File.OpenRead(assemblyPath); + peReader = new PEReader(stream, PEStreamOptions.PrefetchEntireImage); + mdReader = peReader.GetMetadataReader(); + TypeSignatureDecoder = new Entities.TypeSignatureDecoder(this); + + globalNamespace = new Lazy(() => Populate(new Entities.Namespace(this, GetId(""), null))); + systemNamespace = new Lazy(() => Populate(new Entities.Namespace(this, "System"))); + genericHandleFactory = new CachedFunction(CreateGenericHandle); + namespaceFactory = new CachedFunction(n => CreateNamespace(mdReader.GetString(n))); + namespaceDefinitionFactory = new CachedFunction(CreateNamespace); + sourceFiles = new CachedFunction(path => new Entities.PdbSourceFile(this, path)); + folders = new CachedFunction(path => new Entities.Folder(this, path)); + sourceLocations = new CachedFunction(location => new Entities.PdbSourceLocation(this, location)); + + defaultGenericContext = new EmptyContext(this); + + var def = mdReader.GetAssemblyDefinition(); + AssemblyPrefix = GetId(def.Name) + "_" + def.Version.ToString() + "::"; + + if (extractPdbs) + { + pdb = PDB.PdbReader.Create(assemblyPath, peReader); + if (pdb != null) + { + cx.Extractor.Logger.Log(Util.Logging.Severity.Info, string.Format("Found PDB information for {0}", assemblyPath)); + } + } + } + + void IDisposable.Dispose() + { + if (pdb != null) + pdb.Dispose(); + peReader.Dispose(); + stream.Dispose(); + } + + /// + /// Extract the contents of a given entity. + /// + /// The entity to extract. + public void Extract(IExtractedEntity entity) + { + foreach (var content in entity.Contents) + { + content.Extract(this); + } + } + + public readonly Id AssemblyPrefix; + + public readonly Entities.TypeSignatureDecoder TypeSignatureDecoder; + + /// + /// A type used to signify something we can't handle yet. + /// Specifically, function pointers (used in C++). + /// + public Entities.Type ErrorType + { + get + { + var errorType = new Entities.ErrorType(this); + Populate(errorType); + return errorType; + } + } + + /// + /// Attempt to locate debugging information for a particular method. + /// + /// Returns null on failure, for example if there was no PDB information found for the + /// DLL, or if the particular method is compiler generated or doesn't come from source code. + /// + /// The handle of the method. + /// The debugging information, or null if the information could not be located. + public PDB.IMethod GetMethodDebugInformation(MethodDefinitionHandle handle) + { + return pdb == null ? null : pdb.GetMethod(handle.ToDebugInformationHandle()); + } + } + + /// + /// When we decode a type/method signature, we need access to + /// generic parameters. + /// + public abstract class GenericContext + { + public Context cx; + + public GenericContext(Context cx) + { + this.cx = cx; + } + + /// + /// The list of generic type parameters. + /// + public abstract IEnumerable TypeParameters { get; } + + /// + /// The list of generic method parameters. + /// + public abstract IEnumerable MethodParameters { get; } + + /// + /// Gets the `p`th type parameter. + /// + /// The index of the parameter. + /// + /// For constructed types, the supplied type. + /// For unbound types, the type parameter. + /// + public Entities.Type GetGenericTypeParameter(int p) + { + return TypeParameters.ElementAt(p); + } + + /// + /// Gets the `p`th method type parameter. + /// + /// The index of the parameter. + /// + /// For constructed types, the supplied type. + /// For unbound types, the type parameter. + /// + public Entities.Type GetGenericMethodParameter(int p) + { + return MethodParameters.ElementAt(p); + } + } + + /// + /// A generic context which does not contain any type parameters. + /// + public class EmptyContext : GenericContext + { + public EmptyContext(Context cx) : base(cx) + { + } + + public override IEnumerable TypeParameters { get { yield break; } } + + public override IEnumerable MethodParameters { get { yield break; } } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs new file mode 100644 index 00000000000..31745a434f6 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs @@ -0,0 +1,152 @@ +using System.Reflection; +using System.Globalization; +using System.Collections.Generic; +using Semmle.Util.Logging; +using System; + +namespace Semmle.Extraction.CIL.Entities +{ + public interface ILocation : IEntity + { + } + + interface IAssembly : ILocation + { + } + + /// + /// An assembly to extract. + /// + public class Assembly : LabelledEntity, IAssembly + { + public override Id IdSuffix => suffix; + + readonly File file; + readonly AssemblyName assemblyName; + + public Assembly(Context cx) : base(cx) + { + cx.assembly = this; + var def = cx.mdReader.GetAssemblyDefinition(); + + assemblyName = new AssemblyName(); + assemblyName.Name = cx.mdReader.GetString(def.Name); + assemblyName.Version = def.Version; + assemblyName.CultureInfo = new CultureInfo(cx.mdReader.GetString(def.Culture)); + + if (!def.PublicKey.IsNil) + assemblyName.SetPublicKey(cx.mdReader.GetBlobBytes(def.PublicKey)); + + ShortId = cx.GetId(assemblyName.FullName) + "#file:///" + cx.assemblyPath.Replace("\\", "/"); + + file = new File(cx, cx.assemblyPath); + } + + static readonly Id suffix = new StringId(";assembly"); + + public override IEnumerable Contents + { + get + { + yield return file; + yield return Tuples.assemblies(this, file, assemblyName.FullName, assemblyName.Name, assemblyName.Version.ToString()); + + if (cx.pdb != null) + { + foreach (var f in cx.pdb.SourceFiles) + { + yield return cx.CreateSourceFile(f); + } + } + + foreach (var handle in cx.mdReader.TypeDefinitions) + { + IExtractionProduct product = null; + try + { + product = cx.Create(handle); + } + catch (InternalError e) + { + cx.cx.Extractor.Message(new Message + { + exception = e, + message = "Error processing type definition", + severity = Semmle.Util.Logging.Severity.Error + }); + } + + // Limitation of C#: Cannot yield return inside a try-catch. + if (product != null) + yield return product; + } + + foreach (var handle in cx.mdReader.MethodDefinitions) + { + IExtractionProduct product = null; + try + { + product = cx.Create(handle); + } + catch (InternalError e) + { + cx.cx.Extractor.Message(new Message + { + exception = e, + message = "Error processing bytecode", + severity = Semmle.Util.Logging.Severity.Error + }); + } + + if (product != null) + yield return product; + } + } + } + + static void ExtractCIL(Extraction.Context cx, string assemblyPath, bool extractPdbs) + { + using (var cilContext = new Context(cx, assemblyPath, extractPdbs)) + { + cilContext.Populate(new Assembly(cilContext)); + cilContext.cx.PopulateAll(); + } + } + + /// + /// Main entry point to the CIL extractor. + /// Call this to extract a given assembly. + /// + /// The trap layout. + /// The full path of the assembly to extract. + /// The logger. + /// True to overwrite existing trap file. + /// Whether to extract PDBs. + /// The path of the trap file. + /// Whether the file was extracted (false=cached). + public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger, bool nocache, bool extractPdbs, out string trapFile, out bool extracted) + { + trapFile = ""; + extracted = false; + try + { + var extractor = new Extractor(false, assemblyPath, logger); + var project = layout.LookupProjectOrDefault(assemblyPath); + using (var trapWriter = project.CreateTrapWriter(logger, assemblyPath + ".cil", true)) + { + trapFile = trapWriter.TrapFile; + if (nocache || !System.IO.File.Exists(trapFile)) + { + var cx = new Extraction.Context(extractor, null, trapWriter, null); + ExtractCIL(cx, assemblyPath, extractPdbs); + extracted = true; + } + } + } + catch (Exception ex) + { + logger.Log(Severity.Error, string.Format("Exception extracting {0}: {1}", assemblyPath, ex)); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Attribute.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Attribute.cs new file mode 100644 index 00000000000..774de6cf145 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Attribute.cs @@ -0,0 +1,100 @@ +using System; +using System.Collections.Generic; +using System.Reflection.Metadata; + +namespace Semmle.Extraction.CIL.Entities +{ + /// + /// A CIL attribute. + /// + interface IAttribute : IExtractedEntity + { + } + + /// + /// Entity representing a CIL attribute. + /// + class Attribute : UnlabelledEntity, IAttribute + { + readonly CustomAttribute attrib; + readonly IEntity @object; + + public Attribute(Context cx, IEntity @object, CustomAttributeHandle handle) : base(cx) + { + attrib = cx.mdReader.GetCustomAttribute(handle); + this.@object = @object; + } + + public override IEnumerable Contents + { + get + { + var constructor = (Method)cx.Create(attrib.Constructor); + yield return constructor; + + yield return Tuples.cil_attribute(this, @object, constructor); + + CustomAttributeValue decoded; + + try + { + decoded = attrib.DecodeValue(new CustomAttributeDecoder(cx)); + } + catch (NotImplementedException) + { + // Attribute decoding is only partial at this stage. + yield break; + } + + for (int index = 0; index < decoded.FixedArguments.Length; ++index) + { + object value = decoded.FixedArguments[index].Value; + yield return Tuples.cil_attribute_positional_argument(this, index, value == null ? "null" : value.ToString()); + } + + foreach (var p in decoded.NamedArguments) + { + object value = p.Value; + yield return Tuples.cil_attribute_named_argument(this, p.Name, value == null ? "null" : value.ToString()); + } + } + } + + public static IEnumerable Populate(Context cx, IEntity @object, CustomAttributeHandleCollection attributes) + { + foreach (var attrib in attributes) + { + yield return new Attribute(cx, @object, attrib); + } + } + } + + /// + /// Helper class to decode the attribute structure. + /// Note that there are some unhandled cases that should be fixed in due course. + /// + class CustomAttributeDecoder : ICustomAttributeTypeProvider + { + readonly Context cx; + public CustomAttributeDecoder(Context cx) { this.cx = cx; } + + public Type GetPrimitiveType(PrimitiveTypeCode typeCode) => cx.Populate(new PrimitiveType(cx, typeCode)); + + public Type GetSystemType() => throw new NotImplementedException(); + + public Type GetSZArrayType(Type elementType) => + cx.Populate(new ArrayType(cx, elementType)); + + public Type GetTypeFromDefinition(MetadataReader reader, TypeDefinitionHandle handle, byte rawTypeKind) => + (Type)cx.Create(handle); + + public Type GetTypeFromReference(MetadataReader reader, TypeReferenceHandle handle, byte rawTypeKind) => + (Type)cx.Create(handle); + + public Type GetTypeFromSerializedName(string name) => throw new NotImplementedException(); + + public PrimitiveTypeCode GetUnderlyingEnumType(Type type) => throw new NotImplementedException(); + + public bool IsSystemType(Type type) => type is PrimitiveType; // ?? + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Event.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Event.cs new file mode 100644 index 00000000000..009afc1b1e9 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Event.cs @@ -0,0 +1,67 @@ +using System.Collections.Generic; +using System.Reflection.Metadata; + +namespace Semmle.Extraction.CIL.Entities +{ + /// + /// An event. + /// + interface IEvent : ILabelledEntity + { + } + + /// + /// An event entity. + /// + class Event : LabelledEntity, IEvent + { + readonly Type parent; + readonly EventDefinition ed; + static readonly Id suffix = CIL.Id.Create(";cil-event"); + + public Event(Context cx, Type parent, EventDefinitionHandle handle) : base(cx) + { + this.parent = parent; + ed = cx.mdReader.GetEventDefinition(handle); + ShortId = parent.ShortId + cx.Dot + cx.ShortName(ed.Name) + suffix; + } + + public override IEnumerable Contents + { + get + { + var signature = (Type)cx.CreateGeneric(parent, ed.Type); + yield return signature; + + yield return Tuples.cil_event(this, parent, cx.ShortName(ed.Name), signature); + + var accessors = ed.GetAccessors(); + if (!accessors.Adder.IsNil) + { + var adder = (Method)cx.CreateGeneric(parent, accessors.Adder); + yield return adder; + yield return Tuples.cil_adder(this, adder); + } + + if (!accessors.Remover.IsNil) + { + var remover = (Method)cx.CreateGeneric(parent, accessors.Remover); + yield return remover; + yield return Tuples.cil_remover(this, remover); + } + + if (!accessors.Raiser.IsNil) + { + var raiser = (Method)cx.CreateGeneric(parent, accessors.Raiser); + yield return raiser; + yield return Tuples.cil_raiser(this, raiser); + } + + foreach (var c in Attribute.Populate(cx, this, ed.GetCustomAttributes())) + yield return c; + } + } + + public override Id IdSuffix => suffix; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/ExceptionRegion.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/ExceptionRegion.cs new file mode 100644 index 00000000000..57035d993af --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/ExceptionRegion.cs @@ -0,0 +1,63 @@ +using System.Collections.Generic; + +namespace Semmle.Extraction.CIL.Entities +{ + interface IExceptionRegion : IExtractedEntity + { + } + + /// + /// An exception region entity. + /// + class ExceptionRegion : UnlabelledEntity, IExceptionRegion + { + readonly GenericContext gc; + readonly MethodImplementation method; + readonly int index; + readonly System.Reflection.Metadata.ExceptionRegion r; + readonly Dictionary jump_table; + + public ExceptionRegion(GenericContext gc, MethodImplementation method, int index, System.Reflection.Metadata.ExceptionRegion r, Dictionary jump_table) : base(gc.cx) + { + this.gc = gc; + this.method = method; + this.index = index; + this.r = r; + this.jump_table = jump_table; + } + + public override IEnumerable Contents + { + get + { + IInstruction try_start, try_end, handler_start; + + if (!jump_table.TryGetValue(r.TryOffset, out try_start)) + throw new InternalError("Failed to retrieve handler"); + if (!jump_table.TryGetValue(r.TryOffset + r.TryLength, out try_end)) + throw new InternalError("Failed to retrieve handler"); + if (!jump_table.TryGetValue(r.HandlerOffset, out handler_start)) + throw new InternalError("Failed to retrieve handler"); + + + yield return Tuples.cil_handler(this, method, index, (int)r.Kind, try_start, try_end, handler_start); + + if (r.FilterOffset != -1) + { + IInstruction filter_start; + if (!jump_table.TryGetValue(r.FilterOffset, out filter_start)) + throw new InternalError("ExceptionRegion filter clause"); + + yield return Tuples.cil_handler_filter(this, filter_start); + } + + if (!r.CatchType.IsNil) + { + var catchType = (Type)cx.CreateGeneric(gc, r.CatchType); + yield return catchType; + yield return Tuples.cil_handler_type(this, catchType); + } + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Field.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Field.cs new file mode 100644 index 00000000000..64367ca9ae0 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Field.cs @@ -0,0 +1,147 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.CodeAnalysis; +using System.Reflection.Metadata; +using System.Reflection; + +namespace Semmle.Extraction.CIL.Entities +{ + /// + /// An entity represting a member. + /// Used to type tuples correctly. + /// + interface IMember : ILabelledEntity + { + } + + /// + /// An entity representing a field. + /// + interface IField : IMember + { + } + + /// + /// An entity representing a field. + /// + abstract class Field : GenericContext, IField + { + protected Field(Context cx) : base(cx) + { + } + + public bool NeedsPopulation { get { return true; } } + + public Label Label { get; set; } + + public IId Id => ShortId + IdSuffix; + + public Id IdSuffix => fieldSuffix; + + static readonly StringId fieldSuffix = new StringId(";cil-field"); + + public Id ShortId + { + get; set; + } + + public abstract Id Name { get; } + + public abstract Type DeclaringType { get; } + + public Location ReportingLocation => throw new NotImplementedException(); + + abstract public Type Type { get; } + + public virtual IEnumerable Contents + { + get + { + yield return Tuples.cil_field(this, DeclaringType, Name.Value, Type); + } + } + + public void Extract(Context cx) + { + cx.Populate(this); + } + + TrapStackBehaviour IEntity.TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } + + sealed class DefinitionField : Field + { + readonly FieldDefinition fd; + readonly GenericContext gc; + + public DefinitionField(GenericContext gc, FieldDefinitionHandle handle) : base(gc.cx) + { + this.gc = gc; + fd = cx.mdReader.GetFieldDefinition(handle); + ShortId = DeclaringType.ShortId + cx.Dot + Name; + } + + public override IEnumerable Contents + { + get + { + foreach (var c in base.Contents) + yield return c; + + if (fd.Attributes.HasFlag(FieldAttributes.Private)) + yield return Tuples.cil_private(this); + + if (fd.Attributes.HasFlag(FieldAttributes.Public)) + yield return Tuples.cil_public(this); + + if (fd.Attributes.HasFlag(FieldAttributes.Family)) + yield return Tuples.cil_protected(this); + + if (fd.Attributes.HasFlag(FieldAttributes.Static)) + yield return Tuples.cil_static(this); + + if (fd.Attributes.HasFlag(FieldAttributes.Assembly)) + yield return Tuples.cil_internal(this); + + foreach (var c in Attribute.Populate(cx, this, fd.GetCustomAttributes())) + yield return c; + } + } + + public override Id Name => cx.GetId(fd.Name); + + public override Type DeclaringType => (Type)cx.Create(fd.GetDeclaringType()); + + public override Type Type => fd.DecodeSignature(cx.TypeSignatureDecoder, DeclaringType); + + public override IEnumerable TypeParameters => throw new NotImplementedException(); + + public override IEnumerable MethodParameters => throw new NotImplementedException(); + } + + sealed class MemberReferenceField : Field + { + readonly MemberReference mr; + readonly GenericContext gc; + readonly Type declType; + + public MemberReferenceField(GenericContext gc, MemberReferenceHandle handle) : base(gc.cx) + { + this.gc = gc; + mr = cx.mdReader.GetMemberReference(handle); + declType = (Type)cx.CreateGeneric(gc, mr.Parent); + ShortId = declType.ShortId + cx.Dot + Name; + } + + public override Id Name => cx.GetId(mr.Name); + + public override Type DeclaringType => declType; + + public override Type Type => mr.DecodeFieldSignature(cx.TypeSignatureDecoder, this); + + public override IEnumerable TypeParameters => gc.TypeParameters.Concat(declType.TypeParameters); + + public override IEnumerable MethodParameters => gc.MethodParameters.Concat(declType.MethodParameters); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs new file mode 100644 index 00000000000..de8f890312a --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs @@ -0,0 +1,66 @@ +using System.Collections.Generic; + +namespace Semmle.Extraction.CIL.Entities +{ + interface IFileOrFolder : IEntity + { + } + + interface IFile : IFileOrFolder + { + } + + public class File : LabelledEntity, IFile + { + protected readonly string path; + + public File(Context cx, string path) : base(cx) + { + this.path = path.Replace("\\", "/"); + ShortId = new StringId(path.Replace(":", "_")); + } + + public override IEnumerable Contents + { + get + { + var parent = cx.CreateFolder(System.IO.Path.GetDirectoryName(path)); + yield return parent; + yield return Tuples.containerparent(parent, this); + yield return Tuples.files(this, path, System.IO.Path.GetFileNameWithoutExtension(path), System.IO.Path.GetExtension(path).Substring(1)); + } + } + + public override Id IdSuffix => suffix; + + static readonly Id suffix = new StringId(";sourcefile"); + } + + public class PdbSourceFile : File + { + readonly PDB.ISourceFile file; + + public PdbSourceFile(Context cx, PDB.ISourceFile file) : base(cx, file.Path) + { + this.file = file; + } + + public override IEnumerable Contents + { + get + { + foreach (var c in base.Contents) + yield return c; + + var text = file.Contents; + + if (text == null) + cx.cx.Extractor.Logger.Log(Util.Logging.Severity.Warning, string.Format("PDB source file {0} could not be found", path)); + else + cx.cx.TrapWriter.Archive(path, text); + + yield return Tuples.file_extraction_mode(this, 2); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs new file mode 100644 index 00000000000..0ed1b51dd99 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs @@ -0,0 +1,42 @@ +using System.Collections.Generic; +using System.IO; + +namespace Semmle.Extraction.CIL.Entities +{ + interface IFolder : IFileOrFolder + { + } + + public class Folder : LabelledEntity, IFolder + { + readonly string path; + + public Folder(Context cx, string path) : base(cx) + { + this.path = path; + ShortId = new StringId(path.Replace("\\", "/").Replace(":", "_")); + } + + static readonly Id suffix = new StringId(";folder"); + + public override IEnumerable Contents + { + get + { + // On Posix, we could get a Windows directory of the form "C:" + bool windowsDriveLetter = path.Length == 2 && char.IsLetter(path[0]) && path[1] == ':'; + + var parent = Path.GetDirectoryName(path); + if (parent != null && !windowsDriveLetter) + { + var parentFolder = cx.CreateFolder(parent); + yield return parentFolder; + yield return Tuples.containerparent(parentFolder, this); + } + yield return Tuples.folders(this, path, Path.GetFileName(path)); + } + } + + public override Id IdSuffix => suffix; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Instruction.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Instruction.cs new file mode 100644 index 00000000000..de6640235f3 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Instruction.cs @@ -0,0 +1,486 @@ +using System; +using System.Collections.Generic; +using System.Reflection.Metadata; +using System.Reflection.Metadata.Ecma335; + +namespace Semmle.Extraction.CIL.Entities +{ + /// + /// A CIL instruction. + /// + interface IInstruction : IExtractedEntity + { + /// + /// Gets the extraction products for branches. + /// + /// The map from offset to instruction. + /// The extraction products. + IEnumerable JumpContents(Dictionary jump_table); + } + + /// + /// A CIL instruction. + /// + class Instruction : UnlabelledEntity, IInstruction + { + /// + /// The additional data following the opcode, if any. + /// + public enum Payload + { + None, TypeTok, Field, Target8, Class, + Method, Arg8, Local8, Target32, Int8, + Int16, Int32, Int64, Float32, Float64, + CallSiteDesc, Switch, String, Constructor, ValueType, + Type, Arg16, Ignore8, Token, Local16, MethodRef + } + + /// + /// For each Payload, how many additional bytes in the bytestream need to be read. + /// + internal static readonly int[] payloadSizes = { + 0, 4, 4, 1, 4, + 4, 1, 1, 4, 1, + 2, 4, 8, 4, 8, + 4, -1, 4, 4, 4, + 4, 2, 1, 4, 2, 4 }; + + // Maps opcodes to payloads for each instruction. + public static readonly Dictionary opPayload = new Dictionary() + { + { ILOpCode.Nop, Payload.None }, + { ILOpCode.Break, Payload.None }, + { ILOpCode.Ldarg_0, Payload.None }, + { ILOpCode.Ldarg_1, Payload.None }, + { ILOpCode.Ldarg_2, Payload.None }, + { ILOpCode.Ldarg_3, Payload.None }, + { ILOpCode.Ldloc_0, Payload.None }, + { ILOpCode.Ldloc_1, Payload.None }, + { ILOpCode.Ldloc_2, Payload.None }, + { ILOpCode.Ldloc_3, Payload.None }, + { ILOpCode.Stloc_0, Payload.None }, + { ILOpCode.Stloc_1, Payload.None }, + { ILOpCode.Stloc_2, Payload.None }, + { ILOpCode.Stloc_3, Payload.None }, + { ILOpCode.Ldarg_s, Payload.Arg8 }, + { ILOpCode.Ldarga_s, Payload.Arg8 }, + { ILOpCode.Starg_s, Payload.Arg8 }, + { ILOpCode.Ldloc_s, Payload.Local8 }, + { ILOpCode.Ldloca_s, Payload.Local8 }, + { ILOpCode.Stloc_s, Payload.Local8 }, + { ILOpCode.Ldnull, Payload.None }, + { ILOpCode.Ldc_i4_m1, Payload.None }, + { ILOpCode.Ldc_i4_0, Payload.None }, + { ILOpCode.Ldc_i4_1, Payload.None }, + { ILOpCode.Ldc_i4_2, Payload.None }, + { ILOpCode.Ldc_i4_3, Payload.None }, + { ILOpCode.Ldc_i4_4, Payload.None }, + { ILOpCode.Ldc_i4_5, Payload.None }, + { ILOpCode.Ldc_i4_6, Payload.None }, + { ILOpCode.Ldc_i4_7, Payload.None }, + { ILOpCode.Ldc_i4_8, Payload.None }, + { ILOpCode.Ldc_i4_s, Payload.Int8 }, + { ILOpCode.Ldc_i4, Payload.Int32 }, + { ILOpCode.Ldc_i8, Payload.Int64 }, + { ILOpCode.Ldc_r4, Payload.Float32 }, + { ILOpCode.Ldc_r8, Payload.Float64 }, + { ILOpCode.Dup, Payload.None }, + { ILOpCode.Pop, Payload.None }, + { ILOpCode.Jmp, Payload.Method }, + { ILOpCode.Call, Payload.Method }, + { ILOpCode.Calli, Payload.CallSiteDesc }, + { ILOpCode.Ret, Payload.None }, + { ILOpCode.Br_s, Payload.Target8 }, + { ILOpCode.Brfalse_s, Payload.Target8 }, + { ILOpCode.Brtrue_s, Payload.Target8 }, + { ILOpCode.Beq_s, Payload.Target8 }, + { ILOpCode.Bge_s, Payload.Target8 }, + { ILOpCode.Bgt_s, Payload.Target8 }, + { ILOpCode.Ble_s, Payload.Target8 }, + { ILOpCode.Blt_s, Payload.Target8 }, + { ILOpCode.Bne_un_s, Payload.Target8 }, + { ILOpCode.Bge_un_s, Payload.Target8 }, + { ILOpCode.Bgt_un_s, Payload.Target8 }, + { ILOpCode.Ble_un_s, Payload.Target8 }, + { ILOpCode.Blt_un_s, Payload.Target8 }, + { ILOpCode.Br, Payload.Target32 }, + { ILOpCode.Brfalse, Payload.Target32 }, + { ILOpCode.Brtrue, Payload.Target32 }, + { ILOpCode.Beq, Payload.Target32 }, + { ILOpCode.Bge, Payload.Target32 }, + { ILOpCode.Bgt, Payload.Target32 }, + { ILOpCode.Ble, Payload.Target32 }, + { ILOpCode.Blt, Payload.Target32 }, + { ILOpCode.Bne_un, Payload.Target32 }, + { ILOpCode.Bge_un, Payload.Target32 }, + { ILOpCode.Bgt_un, Payload.Target32 }, + { ILOpCode.Ble_un, Payload.Target32 }, + { ILOpCode.Blt_un, Payload.Target32 }, + { ILOpCode.Switch, Payload.Switch }, + { ILOpCode.Ldind_i1, Payload.None }, + { ILOpCode.Ldind_u1, Payload.None }, + { ILOpCode.Ldind_i2, Payload.None }, + { ILOpCode.Ldind_u2, Payload.None }, + { ILOpCode.Ldind_i4, Payload.None }, + { ILOpCode.Ldind_u4, Payload.None }, + { ILOpCode.Ldind_i8, Payload.None }, + { ILOpCode.Ldind_i, Payload.None }, + { ILOpCode.Ldind_r4, Payload.None }, + { ILOpCode.Ldind_r8, Payload.None }, + { ILOpCode.Ldind_ref, Payload.None }, + { ILOpCode.Stind_ref, Payload.None }, + { ILOpCode.Stind_i1, Payload.None }, + { ILOpCode.Stind_i2, Payload.None }, + { ILOpCode.Stind_i4, Payload.None }, + { ILOpCode.Stind_i8, Payload.None }, + { ILOpCode.Stind_r4, Payload.None }, + { ILOpCode.Stind_r8, Payload.None }, + { ILOpCode.Add, Payload.None }, + { ILOpCode.Sub, Payload.None }, + { ILOpCode.Mul, Payload.None }, + { ILOpCode.Div, Payload.None }, + { ILOpCode.Div_un, Payload.None }, + { ILOpCode.Rem, Payload.None }, + { ILOpCode.Rem_un, Payload.None }, + { ILOpCode.And, Payload.None }, + { ILOpCode.Or, Payload.None }, + { ILOpCode.Xor, Payload.None }, + { ILOpCode.Shl, Payload.None }, + { ILOpCode.Shr, Payload.None }, + { ILOpCode.Shr_un, Payload.None }, + { ILOpCode.Neg, Payload.None }, + { ILOpCode.Not, Payload.None }, + { ILOpCode.Conv_i1, Payload.None }, + { ILOpCode.Conv_i2, Payload.None }, + { ILOpCode.Conv_i4, Payload.None }, + { ILOpCode.Conv_i8, Payload.None }, + { ILOpCode.Conv_r4, Payload.None }, + { ILOpCode.Conv_r8, Payload.None }, + { ILOpCode.Conv_u4, Payload.None }, + { ILOpCode.Conv_u8, Payload.None }, + { ILOpCode.Callvirt, Payload.MethodRef }, + { ILOpCode.Cpobj, Payload.TypeTok }, + { ILOpCode.Ldobj, Payload.TypeTok }, + { ILOpCode.Ldstr, Payload.String }, + { ILOpCode.Newobj, Payload.Constructor }, + { ILOpCode.Castclass, Payload.Class }, + { ILOpCode.Isinst, Payload.Class }, + { ILOpCode.Conv_r_un, Payload.None }, + { ILOpCode.Unbox, Payload.ValueType }, + { ILOpCode.Throw, Payload.None }, + { ILOpCode.Ldfld, Payload.Field }, + { ILOpCode.Ldflda, Payload.Field }, + { ILOpCode.Stfld, Payload.Field }, + { ILOpCode.Ldsfld, Payload.Field }, + { ILOpCode.Ldsflda, Payload.Field }, + { ILOpCode.Stsfld, Payload.Field }, + { ILOpCode.Stobj, Payload.Field }, + { ILOpCode.Conv_ovf_i1_un, Payload.None }, + { ILOpCode.Conv_ovf_i2_un, Payload.None }, + { ILOpCode.Conv_ovf_i4_un, Payload.None }, + { ILOpCode.Conv_ovf_i8_un, Payload.None }, + { ILOpCode.Conv_ovf_u1_un, Payload.None }, + { ILOpCode.Conv_ovf_u2_un, Payload.None }, + { ILOpCode.Conv_ovf_u4_un, Payload.None }, + { ILOpCode.Conv_ovf_u8_un, Payload.None }, + { ILOpCode.Conv_ovf_i_un, Payload.None }, + { ILOpCode.Conv_ovf_u_un, Payload.None }, + { ILOpCode.Box, Payload.TypeTok }, + { ILOpCode.Newarr, Payload.TypeTok }, + { ILOpCode.Ldlen, Payload.None }, + { ILOpCode.Ldelema, Payload.Class }, + { ILOpCode.Ldelem_i1, Payload.None }, + { ILOpCode.Ldelem_u1, Payload.None }, + { ILOpCode.Ldelem_i2, Payload.None }, + { ILOpCode.Ldelem_u2, Payload.None }, + { ILOpCode.Ldelem_i4, Payload.None }, + { ILOpCode.Ldelem_u4, Payload.None }, + { ILOpCode.Ldelem_i8, Payload.None }, + { ILOpCode.Ldelem_i, Payload.None }, + { ILOpCode.Ldelem_r4, Payload.None }, + { ILOpCode.Ldelem_r8, Payload.None }, + { ILOpCode.Ldelem_ref, Payload.None }, + { ILOpCode.Stelem_i, Payload.None }, + { ILOpCode.Stelem_i1, Payload.None }, + { ILOpCode.Stelem_i2, Payload.None }, + { ILOpCode.Stelem_i4, Payload.None }, + { ILOpCode.Stelem_i8, Payload.None }, + { ILOpCode.Stelem_r4, Payload.None }, + { ILOpCode.Stelem_r8, Payload.None }, + { ILOpCode.Stelem_ref, Payload.None }, + { ILOpCode.Ldelem, Payload.TypeTok }, + { ILOpCode.Stelem, Payload.TypeTok }, + { ILOpCode.Unbox_any, Payload.TypeTok }, + { ILOpCode.Conv_ovf_i1, Payload.None }, + { ILOpCode.Conv_ovf_u1, Payload.None }, + { ILOpCode.Conv_ovf_i2, Payload.None }, + { ILOpCode.Conv_ovf_u2, Payload.None }, + { ILOpCode.Conv_ovf_i4, Payload.None }, + { ILOpCode.Conv_ovf_u4, Payload.None }, + { ILOpCode.Conv_ovf_i8, Payload.None }, + { ILOpCode.Conv_ovf_u8, Payload.None }, + { ILOpCode.Refanyval, Payload.Type }, + { ILOpCode.Ckfinite, Payload.None }, + { ILOpCode.Mkrefany, Payload.Class }, + { ILOpCode.Ldtoken, Payload.Token }, + { ILOpCode.Conv_u2, Payload.None }, + { ILOpCode.Conv_u1, Payload.None }, + { ILOpCode.Conv_i, Payload.None }, + { ILOpCode.Conv_ovf_i, Payload.None }, + { ILOpCode.Conv_ovf_u, Payload.None }, + { ILOpCode.Add_ovf, Payload.None }, + { ILOpCode.Add_ovf_un, Payload.None }, + { ILOpCode.Mul_ovf, Payload.None }, + { ILOpCode.Mul_ovf_un, Payload.None }, + { ILOpCode.Sub_ovf, Payload.None }, + { ILOpCode.Sub_ovf_un, Payload.None }, + { ILOpCode.Endfinally, Payload.None }, + { ILOpCode.Leave, Payload.Target32 }, + { ILOpCode.Leave_s, Payload.Target8 }, + { ILOpCode.Stind_i, Payload.None }, + { ILOpCode.Conv_u, Payload.None }, + { ILOpCode.Arglist, Payload.None }, + { ILOpCode.Ceq, Payload.None }, + { ILOpCode.Cgt, Payload.None }, + { ILOpCode.Cgt_un, Payload.None }, + { ILOpCode.Clt, Payload.None }, + { ILOpCode.Clt_un, Payload.None }, + { ILOpCode.Ldftn, Payload.Method }, + { ILOpCode.Ldvirtftn, Payload.Method }, + { ILOpCode.Ldarg, Payload.Arg16 }, + { ILOpCode.Ldarga, Payload.Arg16 }, + { ILOpCode.Starg, Payload.Arg16 }, + { ILOpCode.Ldloc, Payload.Local16 }, + { ILOpCode.Ldloca, Payload.Local16 }, + { ILOpCode.Stloc, Payload.Local16 }, + { ILOpCode.Localloc, Payload.None }, + { ILOpCode.Endfilter, Payload.None }, + { ILOpCode.Unaligned, Payload.Ignore8 }, + { ILOpCode.Volatile, Payload.None }, + { ILOpCode.Tail, Payload.None }, + { ILOpCode.Initobj, Payload.TypeTok }, + { ILOpCode.Constrained, Payload.Type }, + { ILOpCode.Cpblk, Payload.None }, + { ILOpCode.Initblk, Payload.None }, + { ILOpCode.Rethrow, Payload.None }, + { ILOpCode.Sizeof, Payload.TypeTok }, + { ILOpCode.Refanytype, Payload.None }, + { ILOpCode.Readonly, Payload.None } + }; + + public readonly DefinitionMethod Method; + public readonly ILOpCode OpCode; + public readonly int Offset; + public readonly int Index; + readonly int PayloadValue; + readonly uint UnsignedPayloadValue; + + public Payload PayloadType + { + get + { + Payload result; + if (!opPayload.TryGetValue(OpCode, out result)) + throw new InternalError("Unknown op code " + OpCode); + return result; + } + } + + public override string ToString() => Index + ": " + OpCode; + + /// + /// The number of bytes of this instruction, + /// including the payload (if any). + /// + public int Width + { + get + { + if (OpCode == ILOpCode.Switch) return 5 + 4 * PayloadValue; + + return ((int)OpCode > 255 ? 2 : 1) + PayloadSize; + } + } + + Label IEntity.Label + { + get; set; + } + + + readonly byte[] data; + + int PayloadSize => payloadSizes[(int)PayloadType]; + + /// + /// Reads the instruction from a byte stream. + /// + /// The byte stream. + /// The offset of the instruction. + /// The index of this instruction in the callable. + public Instruction(Context cx, DefinitionMethod method, byte[] data, int offset, int index) : base(cx) + { + Method = method; + Offset = offset; + Index = index; + this.data = data; + int opcode = data[offset]; + ++offset; + + /* + * An opcode is either 1 or 2 bytes, followed by an optional payload depending on the instruction. + * Instructions where the first byte is 0xfe are 2-byte instructions. + */ + if (opcode == 0xfe) + opcode = opcode << 8 | data[offset++]; + OpCode = (ILOpCode)opcode; + + switch (PayloadSize) + { + case 0: + PayloadValue = 0; + break; + case 1: + PayloadValue = (sbyte)data[offset]; + UnsignedPayloadValue = data[offset]; + break; + case 2: + PayloadValue = BitConverter.ToInt16(data, offset); + UnsignedPayloadValue = BitConverter.ToUInt16(data, offset); + break; + case -1: // Switch + case 4: + PayloadValue = BitConverter.ToInt32(data, offset); + break; + case 8: // Not handled here. + break; + default: + throw new InternalError("Unhandled CIL instruction Payload"); + } + } + + public override IEnumerable Contents + { + get + { + int offset = Offset; + + yield return Tuples.cil_instruction(this, (int)OpCode, Index, Method.Implementation); + + switch (PayloadType) + { + case Payload.String: + yield return Tuples.cil_value(this, cx.mdReader.GetUserString(MetadataTokens.UserStringHandle(PayloadValue))); + break; + case Payload.Float32: + yield return Tuples.cil_value(this, BitConverter.ToSingle(data, offset).ToString()); + break; + case Payload.Float64: + yield return Tuples.cil_value(this, BitConverter.ToDouble(data, offset).ToString()); + break; + case Payload.Int8: + yield return Tuples.cil_value(this, data[offset].ToString()); + break; + case Payload.Int16: + yield return Tuples.cil_value(this, BitConverter.ToInt16(data, offset).ToString()); + break; + case Payload.Int32: + yield return Tuples.cil_value(this, BitConverter.ToInt32(data, offset).ToString()); + break; + case Payload.Int64: + yield return Tuples.cil_value(this, BitConverter.ToInt64(data, offset).ToString()); + break; + case Payload.Constructor: + case Payload.Method: + case Payload.MethodRef: + case Payload.Class: + case Payload.TypeTok: + case Payload.Token: + case Payload.Type: + case Payload.Field: + case Payload.ValueType: + // A generic EntityHandle. + var handle = MetadataTokens.EntityHandle(PayloadValue); + var target = cx.CreateGeneric(Method, handle); + yield return target; + if (target != null) + { + yield return Tuples.cil_access(this, target); + } + else + { + throw new InternalError("Unable to create payload type {0} for opcode {1}", PayloadType, OpCode); + } + break; + case Payload.Arg8: + case Payload.Arg16: + yield return Tuples.cil_access(this, Method.Parameters[(int)UnsignedPayloadValue]); + break; + case Payload.Local8: + case Payload.Local16: + yield return Tuples.cil_access(this, Method.LocalVariables[(int)UnsignedPayloadValue]); + break; + case Payload.None: + case Payload.Target8: + case Payload.Target32: + case Payload.Switch: + case Payload.Ignore8: + case Payload.CallSiteDesc: + // These are not handled here. + // Some of these are handled by JumpContents(). + break; + default: + throw new InternalError("Unhandled payload type {0}", PayloadType); + } + } + } + + // Called to populate the jumps in each instruction. + public IEnumerable JumpContents(Dictionary jump_table) + { + int target; + IInstruction inst; + + switch (PayloadType) + { + case Payload.Target8: + target = Offset + PayloadValue + 2; + break; + case Payload.Target32: + target = Offset + PayloadValue + 5; + break; + case Payload.Switch: + int end = Offset + Width; + + int offset = Offset + 5; + + for (int b = 0; b < PayloadValue; ++b, offset += 4) + { + target = BitConverter.ToInt32(data, offset) + end; + if (!jump_table.TryGetValue(target, out inst)) + throw new InternalError("Invalid jump target"); + yield return Tuples.cil_switch(this, b, inst); + } + + yield break; + default: + // Not a jump + yield break; + } + + + if (jump_table.TryGetValue(target, out inst)) + { + yield return Tuples.cil_jump(this, inst); + } + else + { + // Sometimes instructions can jump outside the current method. + // TODO: Find a solution to this. + + // For now, just log the error + cx.cx.Extractor.Message(new Message { message = "A CIL instruction jumps outside the current method", severity = Util.Logging.Severity.Warning }); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/LocalVariable.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/LocalVariable.cs new file mode 100644 index 00000000000..4e3fbf43662 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/LocalVariable.cs @@ -0,0 +1,36 @@ +using System.Collections.Generic; + +namespace Semmle.Extraction.CIL.Entities +{ + interface ILocal : ILabelledEntity + { + } + + class LocalVariable : LabelledEntity, ILocal + { + readonly MethodImplementation method; + readonly int index; + readonly Type type; + + public LocalVariable(Context cx, MethodImplementation m, int i, Type t) : base(cx) + { + method = m; + index = i; + type = t; + ShortId = CIL.Id.Create(method.Label) + underscore + index; + } + + static readonly Id underscore = CIL.Id.Create("_"); + static readonly Id suffix = CIL.Id.Create(";cil-local"); + public override Id IdSuffix => suffix; + + public override IEnumerable Contents + { + get + { + yield return type; + yield return Tuples.cil_local_variable(this, method, index, type); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Method.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Method.cs new file mode 100644 index 00000000000..306e2205c0b --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Method.cs @@ -0,0 +1,515 @@ +using System; +using System.Collections.Immutable; +using System.Reflection.Metadata; +using Microsoft.CodeAnalysis; +using System.Collections.Generic; +using System.Reflection; +using System.Linq; + +namespace Semmle.Extraction.CIL.Entities +{ + /// + /// A method entity. + /// + interface IMethod : IMember + { + } + + /// + /// A method entity. + /// + abstract class Method : TypeContainer, IMethod + { + protected Method(GenericContext gc) : base(gc.cx) + { + this.gc = gc; + } + + public override IEnumerable TypeParameters => gc.TypeParameters.Concat(declaringType.TypeParameters); + + public override IEnumerable MethodParameters => genericParams == null ? Enumerable.Empty() : genericParams; + + public int GenericParameterCount => signature.GenericParameterCount; + + public virtual Method SourceDeclaration => this; + + public abstract Type DeclaringType { get; } + public abstract string Name { get; } + + public virtual IList LocalVariables => throw new NotImplementedException(); + public IList Parameters { get; private set; } + + static readonly Id tick = CIL.Id.Create("`"); + static readonly Id space = CIL.Id.Create(" "); + static readonly Id dot = CIL.Id.Create("."); + static readonly Id open = CIL.Id.Create("("); + static readonly Id close = CIL.Id.Create(")"); + + internal protected Id MakeMethodId(Type parent, Id methodName) + { + var id = signature.ReturnType.MakeId(gc) + space + parent.ShortId + dot + methodName; + + if (signature.GenericParameterCount > 0) + { + id += tick + signature.GenericParameterCount; + } + + id += open + CIL.Id.CommaSeparatedList(signature.ParameterTypes.Select(p => p.MakeId(gc))) + close; + return id; + } + + protected MethodTypeParameter[] genericParams; + protected Type declaringType; + protected GenericContext gc; + protected MethodSignature signature; + protected Id name; + + static readonly StringId methodSuffix = new StringId(";cil-method"); + + public override Id IdSuffix => methodSuffix; + + protected void PopulateParameters(IEnumerable parameterTypes) + { + Parameters = MakeParameters(parameterTypes).ToArray(); + } + + protected IEnumerable PopulateFlags + { + get + { + if (IsStatic) + yield return Tuples.cil_static(this); + } + } + + public abstract bool IsStatic { get; } + + private IEnumerable MakeParameters(IEnumerable parameterTypes) + { + int i = 0; + + if (!IsStatic) + { + yield return cx.Populate(new Parameter(cx, this, i++, DeclaringType)); + } + + foreach (var p in parameterTypes) + yield return cx.Populate(new Parameter(cx, this, i++, p)); + } + } + + /// + /// A method implementation entity. + /// + interface IMethodImplementation : IExtractedEntity + { + } + + /// + /// A method implementation entity. + /// In the database, the same method could in principle have multiple implementations. + /// + class MethodImplementation : UnlabelledEntity, IMethodImplementation + { + readonly Method m; + + public MethodImplementation(Method m) : base(m.cx) + { + this.m = m; + } + + public override IEnumerable Contents + { + get + { + yield return Tuples.cil_method_implementation(this, m, cx.assembly); + } + } + } + + + /// + /// A definition method - a method defined in the current assembly. + /// + class DefinitionMethod : Method, IMember + { + readonly MethodDefinition md; + readonly PDB.IMethod methodDebugInformation; + + LocalVariable[] locals; + + public MethodImplementation Implementation { get; private set; } + + public override IList LocalVariables => locals; + + public DefinitionMethod(GenericContext gc, MethodDefinitionHandle handle) : base(gc) + { + md = cx.mdReader.GetMethodDefinition(handle); + this.gc = gc; + name = cx.GetId(md.Name); + + declaringType = (Type)cx.CreateGeneric(this, md.GetDeclaringType()); + + signature = md.DecodeSignature(new SignatureDecoder(), this); + ShortId = MakeMethodId(declaringType, name); + + methodDebugInformation = cx.GetMethodDebugInformation(handle); + } + + public override bool IsStatic => !signature.Header.IsInstance; + + public override Type DeclaringType => declaringType; + + public override string Name => cx.ShortName(md.Name); + + /// + /// Holds if this method has bytecode. + /// + public bool HasBytecode => md.ImplAttributes == MethodImplAttributes.IL && md.RelativeVirtualAddress != 0; + + public override IEnumerable Contents + { + get + { + if (md.GetGenericParameters().Any()) + { + // We need to perform a 2-phase population because some type parameters can + // depend on other type parameters (as a constraint). + genericParams = new MethodTypeParameter[md.GetGenericParameters().Count]; + for (int i = 0; i < genericParams.Length; ++i) + genericParams[i] = cx.Populate(new MethodTypeParameter(this, this, i)); + for (int i = 0; i < genericParams.Length; ++i) + genericParams[i].PopulateHandle(this, md.GetGenericParameters()[i]); + foreach (var p in genericParams) + yield return p; + } + + var typeSignature = md.DecodeSignature(cx.TypeSignatureDecoder, this); + + PopulateParameters(typeSignature.ParameterTypes); + + foreach (var c in Parameters) + yield return c; + + foreach (var c in PopulateFlags) + yield return c; + + foreach (var p in md.GetParameters().Select(h => cx.mdReader.GetParameter(h)).Where(p => p.SequenceNumber > 0)) + { + var pe = Parameters[IsStatic ? p.SequenceNumber - 1 : p.SequenceNumber]; + if (p.Attributes.HasFlag(ParameterAttributes.Out)) + yield return Tuples.cil_parameter_out(pe); + if (p.Attributes.HasFlag(ParameterAttributes.In)) + yield return Tuples.cil_parameter_in(pe); + Attribute.Populate(cx, pe, p.GetCustomAttributes()); + } + + yield return Tuples.cil_method(this, Name, declaringType, typeSignature.ReturnType); + yield return Tuples.cil_method_source_declaration(this, this); + yield return Tuples.cil_method_location(this, cx.assembly); + + if (HasBytecode) + { + Implementation = new MethodImplementation(this); + yield return Implementation; + + var body = cx.peReader.GetMethodBody(md.RelativeVirtualAddress); + + if (!body.LocalSignature.IsNil) + { + var locals = cx.mdReader.GetStandaloneSignature(body.LocalSignature); + var localVariableTypes = locals.DecodeLocalSignature(cx.TypeSignatureDecoder, this); + + this.locals = new LocalVariable[localVariableTypes.Length]; + + for (int l = 0; l < this.locals.Length; ++l) + { + this.locals[l] = cx.Populate(new LocalVariable(cx, Implementation, l, localVariableTypes[l])); + yield return this.locals[l]; + } + } + + var jump_table = new Dictionary(); + + foreach (var c in Decode(body.GetILBytes(), jump_table)) + yield return c; + + int filter_index = 0; + foreach (var region in body.ExceptionRegions) + { + yield return new ExceptionRegion(this, Implementation, filter_index++, region, jump_table); + } + + yield return Tuples.cil_method_stack_size(Implementation, body.MaxStack); + + if (methodDebugInformation != null) + { + var sourceLocation = cx.CreateSourceLocation(methodDebugInformation.Location); + yield return sourceLocation; + yield return Tuples.cil_method_location(this, sourceLocation); + } + } + + // Flags + + if (md.Attributes.HasFlag(MethodAttributes.Private)) + yield return Tuples.cil_private(this); + + if (md.Attributes.HasFlag(MethodAttributes.Public)) + yield return Tuples.cil_public(this); + + if (md.Attributes.HasFlag(MethodAttributes.Family)) + yield return Tuples.cil_protected(this); + + if (md.Attributes.HasFlag(MethodAttributes.Final)) + yield return Tuples.cil_sealed(this); + + if (md.Attributes.HasFlag(MethodAttributes.Virtual)) + yield return Tuples.cil_virtual(this); + + if (md.Attributes.HasFlag(MethodAttributes.Abstract)) + yield return Tuples.cil_abstract(this); + + if (md.Attributes.HasFlag(MethodAttributes.HasSecurity)) + yield return Tuples.cil_security(this); + + if (md.Attributes.HasFlag(MethodAttributes.RequireSecObject)) + yield return Tuples.cil_requiresecobject(this); + + if (md.Attributes.HasFlag(MethodAttributes.SpecialName)) + yield return Tuples.cil_specialname(this); + + if (md.Attributes.HasFlag(MethodAttributes.NewSlot)) + yield return Tuples.cil_newslot(this); + + // Populate attributes + Attribute.Populate(cx, this, md.GetCustomAttributes()); + } + } + + IEnumerable Decode(byte[] ilbytes, Dictionary jump_table) + { + // Sequence points are stored in order of offset. + // We use an enumerator to locate the correct sequence point for each instruction. + // The sequence point gives the location of each instruction. + // The location of an instruction is given by the sequence point *after* the + // instruction. + IEnumerator nextSequencePoint = null; + PdbSourceLocation instructionLocation = null; + + if (methodDebugInformation != null) + { + nextSequencePoint = methodDebugInformation.SequencePoints.GetEnumerator(); + if (nextSequencePoint.MoveNext()) + { + instructionLocation = cx.CreateSourceLocation(nextSequencePoint.Current.Location); + yield return instructionLocation; + } + else + { + nextSequencePoint = null; + } + } + + int child = 0; + for (int offset = 0; offset < ilbytes.Length;) + { + var instruction = new Instruction(cx, this, ilbytes, offset, child++); + yield return instruction; + + if (nextSequencePoint != null && offset >= nextSequencePoint.Current.Offset) + { + instructionLocation = cx.CreateSourceLocation(nextSequencePoint.Current.Location); + yield return instructionLocation; + if (!nextSequencePoint.MoveNext()) + nextSequencePoint = null; + } + + if (instructionLocation != null) + yield return Tuples.cil_instruction_location(instruction, instructionLocation); + + jump_table.Add(instruction.Offset, instruction); + offset += instruction.Width; + } + + foreach (var i in jump_table) + { + foreach (var t in i.Value.JumpContents(jump_table)) + yield return t; + } + } + + /// + /// Display the instructions in the method in the debugger. + /// This is only used for debugging, not in the code itself. + /// + public IEnumerable DebugInstructions + { + get + { + if (md.ImplAttributes == MethodImplAttributes.IL && md.RelativeVirtualAddress != 0) + { + var body = cx.peReader.GetMethodBody(md.RelativeVirtualAddress); + + var ilbytes = body.GetILBytes(); + + int child = 0; + for (int offset = 0; offset < ilbytes.Length;) + { + Instruction decoded; + try + { + decoded = new Instruction(cx, this, ilbytes, offset, child++); + offset += decoded.Width; + } + catch + { + yield break; + } + yield return decoded; + } + } + } + } + } + + /// + /// This is a late-bound reference to a method. + /// + class MemberReferenceMethod : Method + { + readonly MemberReference mr; + readonly Type declType; + readonly GenericContext parent; + readonly Method sourceDeclaration; + + public MemberReferenceMethod(GenericContext gc, MemberReferenceHandle handle) : base(gc) + { + this.gc = gc; + mr = cx.mdReader.GetMemberReference(handle); + + signature = mr.DecodeMethodSignature(new SignatureDecoder(), gc); + + parent = (GenericContext)cx.CreateGeneric(gc, mr.Parent); + + var parentMethod = parent as Method; + var nameLabel = cx.GetId(mr.Name); + + declType = parentMethod == null ? parent as Type : parentMethod.DeclaringType; + + ShortId = MakeMethodId(declType, nameLabel); + + var typeSourceDeclaration = declType.SourceDeclaration; + sourceDeclaration = typeSourceDeclaration == declType ? (Method)this : typeSourceDeclaration.LookupMethod(mr.Name, mr.Signature); + } + + public override Method SourceDeclaration => sourceDeclaration; + + public override bool IsStatic => !signature.Header.IsInstance; + + public override Type DeclaringType => declType; + + public override string Name => cx.ShortName(mr.Name); + + public override IEnumerable TypeParameters => parent.TypeParameters.Concat(gc.TypeParameters); + + public override IEnumerable Contents + { + get + { + genericParams = new MethodTypeParameter[signature.GenericParameterCount]; + for (int p = 0; p < genericParams.Length; ++p) + genericParams[p] = cx.Populate(new MethodTypeParameter(this, this, p)); + + foreach (var p in genericParams) + yield return p; + + var typeSignature = mr.DecodeMethodSignature(cx.TypeSignatureDecoder, this); + + PopulateParameters(typeSignature.ParameterTypes); + foreach (var p in Parameters) yield return p; + + foreach (var f in PopulateFlags) yield return f; + + yield return Tuples.cil_method(this, Name, DeclaringType, typeSignature.ReturnType); + + if (SourceDeclaration != null) + yield return Tuples.cil_method_source_declaration(this, SourceDeclaration); + } + } + } + + /// + /// A constructed method. + /// + class MethodSpecificationMethod : Method + { + readonly MethodSpecification ms; + readonly Method unboundMethod; + readonly ImmutableArray typeParams; + + public MethodSpecificationMethod(GenericContext gc, MethodSpecificationHandle handle) : base(gc) + { + ms = cx.mdReader.GetMethodSpecification(handle); + + typeParams = ms.DecodeSignature(cx.TypeSignatureDecoder, gc); + + unboundMethod = (Method)cx.CreateGeneric(gc, ms.Method); + declaringType = unboundMethod.DeclaringType; + + ShortId = unboundMethod.ShortId + openAngle + CIL.Id.CommaSeparatedList(typeParams.Select(p => p.ShortId)) + closeAngle; + } + + static readonly Id openAngle = CIL.Id.Create("<"); + static readonly Id closeAngle = CIL.Id.Create(">"); + + public override Method SourceDeclaration => unboundMethod; + + public override Type DeclaringType => unboundMethod.DeclaringType; + + public override string Name => unboundMethod.Name; + + public override bool IsStatic => unboundMethod.IsStatic; + + public override IEnumerable MethodParameters => typeParams; + + public override IEnumerable Contents + { + get + { + MethodSignature constructedTypeSignature; + switch (ms.Method.Kind) + { + case HandleKind.MemberReference: + var mr = cx.mdReader.GetMemberReference((MemberReferenceHandle)ms.Method); + constructedTypeSignature = mr.DecodeMethodSignature(cx.TypeSignatureDecoder, this); + break; + case HandleKind.MethodDefinition: + var md = cx.mdReader.GetMethodDefinition((MethodDefinitionHandle)ms.Method); + constructedTypeSignature = md.DecodeSignature(cx.TypeSignatureDecoder, this); + break; + default: + throw new InternalError("Unexpected constructed method handle kind {0}", ms.Method.Kind); + } + + PopulateParameters(constructedTypeSignature.ParameterTypes); + foreach (var p in Parameters) + yield return p; + + foreach (var f in PopulateFlags) + yield return f; + + yield return Tuples.cil_method(this, Name, DeclaringType, constructedTypeSignature.ReturnType); + yield return Tuples.cil_method_source_declaration(this, SourceDeclaration); + + if (typeParams.Count() != unboundMethod.GenericParameterCount) + throw new InternalError("Method type parameter mismatch"); + + for (int p = 0; p < typeParams.Length; ++p) + { + yield return Tuples.cil_type_argument(this, p, typeParams[p]); + } + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs new file mode 100644 index 00000000000..e34d68554e7 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs @@ -0,0 +1,78 @@ +using System; +using System.Collections.Generic; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CIL.Entities +{ + /// + /// A namespace. + /// + interface INamespace : ITypeContainer + { + } + + /// + /// A namespace. + /// + public class Namespace : TypeContainer, INamespace + { + public Namespace ParentNamespace; + public readonly StringId Name; + + public bool IsGlobalNamespace => ParentNamespace == null; + + static readonly Id suffix = CIL.Id.Create(";namespace"); + + public Id CreateId + { + get + { + if (ParentNamespace != null && !ParentNamespace.IsGlobalNamespace) + { + return ParentNamespace.ShortId + cx.Dot + Name; + } + return Name; + } + } + + public override Id IdSuffix => suffix; + + public override IEnumerable TypeParameters => throw new NotImplementedException(); + + public override IEnumerable MethodParameters => throw new NotImplementedException(); + + static string parseNamespaceName(string fqn) + { + var i = fqn.LastIndexOf('.'); + return i == -1 ? fqn : fqn.Substring(i + 1); + } + + static Namespace createParentNamespace(Context cx, string fqn) + { + if (fqn == "") return null; + var i = fqn.LastIndexOf('.'); + return i == -1 ? cx.GlobalNamespace : cx.Populate(new Namespace(cx, fqn.Substring(0, i))); + } + + public Namespace(Context cx, string fqn) : this(cx, cx.GetId(parseNamespaceName(fqn)), createParentNamespace(cx, fqn)) + { + } + + public Namespace(Context cx, StringId name, Namespace parent) : base(cx) + { + Name = name; + ParentNamespace = parent; + ShortId = CreateId; + } + + public override IEnumerable Contents + { + get + { + yield return Tuples.namespaces(this, Name.Value); + if (!IsGlobalNamespace) + yield return Tuples.parent_namespace(this, ParentNamespace); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Parameter.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Parameter.cs new file mode 100644 index 00000000000..b008ccc2510 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Parameter.cs @@ -0,0 +1,43 @@ +using System.Collections.Generic; + +namespace Semmle.Extraction.CIL.Entities +{ + /// + /// A parameter entity. + /// + interface IParameter : ILabelledEntity + { + } + + /// + /// A parameter entity. + /// + class Parameter : LabelledEntity, IParameter + { + readonly Method method; + readonly int index; + readonly Type type; + + public Parameter(Context cx, Method m, int i, Type t) : base(cx) + { + method = m; + index = i; + type = t; + ShortId = openCurly + method.Label.Value + closeCurly + index; + } + + static readonly Id parameterSuffix = CIL.Id.Create(";cil-parameter"); + static readonly Id openCurly = CIL.Id.Create("{#"); + static readonly Id closeCurly = CIL.Id.Create("}_"); + + public override Id IdSuffix => parameterSuffix; + + public override IEnumerable Contents + { + get + { + yield return Tuples.cil_parameter(this, method, index, type); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Property.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Property.cs new file mode 100644 index 00000000000..2adacd2e804 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Property.cs @@ -0,0 +1,64 @@ +using System.Collections.Generic; +using System.Reflection.Metadata; +using System.Linq; + +namespace Semmle.Extraction.CIL.Entities +{ + /// + /// A property. + /// + interface IProperty : ILabelledEntity + { + } + + /// + /// A property. + /// + class Property : LabelledEntity, IProperty + { + readonly Type type; + readonly PropertyDefinition pd; + static readonly Id suffix = CIL.Id.Create(";cil-property"); + + public Property(GenericContext gc, Type type, PropertyDefinitionHandle handle) : base(gc.cx) + { + pd = cx.mdReader.GetPropertyDefinition(handle); + this.type = type; + + var id = type.ShortId + gc.cx.Dot + cx.ShortName(pd.Name); + var signature = pd.DecodeSignature(new SignatureDecoder(), gc); + id += "(" + CIL.Id.CommaSeparatedList(signature.ParameterTypes.Select(p => p.MakeId(gc))) + ")"; + ShortId = id; + } + + public override IEnumerable Contents + { + get + { + var sig = pd.DecodeSignature(cx.TypeSignatureDecoder, type); + + yield return Tuples.cil_property(this, type, cx.ShortName(pd.Name), sig.ReturnType); + + var accessors = pd.GetAccessors(); + if (!accessors.Getter.IsNil) + { + var getter = (Method)cx.CreateGeneric(type, accessors.Getter); + yield return getter; + yield return Tuples.cil_getter(this, getter); + } + + if (!accessors.Setter.IsNil) + { + var setter = (Method)cx.CreateGeneric(type, accessors.Setter); + yield return setter; + yield return Tuples.cil_setter(this, setter); + } + + foreach (var c in Attribute.Populate(cx, this, pd.GetCustomAttributes())) + yield return c; + } + } + + public override Id IdSuffix => suffix; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs new file mode 100644 index 00000000000..88592789099 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs @@ -0,0 +1,37 @@ +using System.Collections.Generic; +using Semmle.Extraction.PDB; + +namespace Semmle.Extraction.CIL.Entities +{ + public interface ISourceLocation : ILocation + { + } + + public sealed class PdbSourceLocation : LabelledEntity, ISourceLocation + { + readonly Location location; + readonly PdbSourceFile file; + + public PdbSourceLocation(Context cx, PDB.Location location) : base(cx) + { + this.location = location; + file = cx.CreateSourceFile(location.File); + + ShortId = file.ShortId + separator + new IntId(location.StartLine) + separator + new IntId(location.StartColumn) + separator + new IntId(location.EndLine) + separator + new IntId(location.EndColumn); + } + + static readonly Id suffix = new StringId(";sourcelocation"); + static readonly Id separator = new StringId(","); + + public override IEnumerable Contents + { + get + { + yield return file; + yield return Tuples.locations_default(this, file, location.StartLine, location.StartColumn, location.EndLine, location.EndColumn); + } + } + + public override Id IdSuffix => suffix; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs new file mode 100644 index 00000000000..ac5c436770f --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs @@ -0,0 +1,1285 @@ +using System; +using Microsoft.CodeAnalysis; +using System.Reflection.Metadata; +using System.Collections.Immutable; +using System.Linq; +using System.Collections.Generic; +using System.Reflection; +using Semmle.Util; + +namespace Semmle.Extraction.CIL.Entities +{ + /// + /// A type. + /// + interface IType : IEntity + { + } + + /// + /// An array type. + /// + interface IArrayType : IType + { + } + + /// + /// The CIL database type-kind. + /// + public enum CilTypeKind + { + ValueOrRefType, + TypeParameter, + Array, + Pointer + } + + /// + /// A type container (namespace/types/method). + /// + interface ITypeContainer : ILabelledEntity + { + } + + /// + /// Base class for all type containers (namespaces, types, methods). + /// + abstract public class TypeContainer : GenericContext, ITypeContainer + { + protected TypeContainer(Context cx) : base(cx) + { + this.cx = cx; + } + + public virtual Label Label { get; set; } + + public virtual IId Id { get { return ShortId + IdSuffix; } } + + public Id ShortId { get; set; } + public abstract Id IdSuffix { get; } + + Location IEntity.ReportingLocation => throw new NotImplementedException(); + + public void Extract(Context cx) { cx.Populate(this); } + + public abstract IEnumerable Contents { get; } + + public override string ToString() + { + return Id.ToString(); + } + + TrapStackBehaviour IEntity.TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } + + /// + /// A type. + /// + public abstract class Type : TypeContainer, IMember, IType + { + static readonly Id suffix = CIL.Id.Create(";cil-type"); + public override Id IdSuffix => suffix; + + /// + /// Find the method in this type matching the name and signature. + /// + /// The handle to the name. + /// + /// The handle to the signature. Note that comparing handles is a valid + /// shortcut to comparing the signature bytes since handles are unique. + /// + /// The method, or 'null' if not found or not supported. + internal virtual Method LookupMethod(StringHandle Name, BlobHandle signature) + { + return null; + } + + public IEnumerable TypeArguments + { + get + { + if (ContainingType != null) + foreach (var t in ContainingType.TypeArguments) + yield return t; + + foreach (var t in ThisTypeArguments) + yield return t; + + } + } + + public virtual IEnumerable ThisTypeArguments + { + get + { + yield break; + } + } + + /// + /// Gets the assembly identifier of this type. + /// + public abstract Id AssemblyPrefix { get; } + + /// + /// Gets the ID part to be used in a method. + /// + /// + /// Whether we should output the context prefix of type parameters. + /// (This is to avoid infinite recursion generating a method ID that returns a + /// type parameter.) + /// + public abstract Id MakeId(bool inContext); + + public Id GetId(bool inContext) + { + return inContext ? MakeId(true) : ShortId; + } + + protected Type(Context cx) : base(cx) { } + + public abstract CilTypeKind Kind + { + get; + } + + public virtual TypeContainer Parent => (TypeContainer)ContainingType ?? Namespace; + + public override IEnumerable Contents + { + get + { + yield return Tuples.cil_type(this, Name.Value, Kind, Parent, SourceDeclaration); + } + } + + /// + /// Whether this type is visible outside the current assembly. + /// + public virtual bool IsVisible => true; + + public abstract Id Name { get; } + + public abstract Namespace Namespace { get; } + + public abstract Type ContainingType { get; } + + public abstract Type Construct(IEnumerable typeArguments); + + /// + /// The number of type arguments, or 0 if this isn't generic. + /// The containing type may also have type arguments. + /// + public abstract int ThisTypeParameters { get; } + + /// + /// The total number of type parameters (including parent types). + /// This is used for internal consistency checking only. + /// + public int TotalTypeParametersCheck => + ContainingType == null ? ThisTypeParameters : ThisTypeParameters + ContainingType.TotalTypeParametersCheck; + + /// + /// Returns all bound/unbound generic arguments + /// of a constructed/unbound generic type. + /// + public virtual IEnumerable ThisGenericArguments + { + get + { + yield break; + } + } + + public virtual IEnumerable GenericArguments + { + get + { + if (ContainingType != null) + foreach (var t in ContainingType.GenericArguments) + yield return t; + foreach (var t in ThisGenericArguments) + yield return t; + } + } + + public virtual Type SourceDeclaration => this; + + protected static readonly Id builtin = CIL.Id.Create("builtin:"); + + public Id PrimitiveTypeId => builtin + Name; + + public bool IsPrimitiveType + { + get + { + if (ContainingType == null && Namespace.ShortId == cx.SystemNamespace.ShortId) + { + switch (Name.Value) + { + case "Boolean": + case "Object": + case "Byte": + case "SByte": + case "Int16": + case "UInt16": + case "Int32": + case "UInt32": + case "Int64": + case "UInt64": + case "Single": + case "Double": + case "String": + case "Void": + case "IntPtr": + case "UIntPtr": + case "Char": + case "TypedReference": + return true; + } + } + return false; + } + } + } + + /// + /// A type defined in the current assembly. + /// + public sealed class TypeDefinitionType : Type + { + readonly TypeDefinition td; + + public TypeDefinitionType(Context cx, TypeDefinitionHandle handle) : base(cx) + { + td = cx.mdReader.GetTypeDefinition(handle); + + declType = + td.GetDeclaringType().IsNil ? null : + (Type)cx.Create(td.GetDeclaringType()); + + ShortId = MakeId(false); + + // Lazy because should happen during population. + typeParams = new Lazy>(MakeTypeParameters); + } + + public override Id MakeId(bool inContext) + { + if (IsPrimitiveType) return PrimitiveTypeId; + + var name = cx.GetId(td.Name); + + Id l; + if (ContainingType != null) + { + l = ContainingType.GetId(inContext) + cx.Dot; + } + else + { + l = AssemblyPrefix; + + var ns = Namespace; + if (!ns.IsGlobalNamespace) + { + l = l + ns.ShortId + cx.Dot; + } + } + + return l + name; + } + + public override Id Name + { + get + { + var name = cx.GetId(td.Name); + var tick = name.Value.IndexOf('`'); + return tick == -1 ? name : cx.GetId(name.Value.Substring(0, tick)); + } + } + + public override Namespace Namespace => cx.Create(td.NamespaceDefinition); + + readonly Type declType; + + public override Type ContainingType => declType; + + public override int ThisTypeParameters + { + get + { + var containingType = td.GetDeclaringType(); + var parentTypeParameters = containingType.IsNil ? 0 : + cx.mdReader.GetTypeDefinition(containingType).GetGenericParameters().Count; + + return td.GetGenericParameters().Count - parentTypeParameters; + } + } + + public override CilTypeKind Kind => CilTypeKind.ValueOrRefType; + + public override Type Construct(IEnumerable typeArguments) + { + return cx.Populate(new ConstructedType(cx, this, typeArguments)); + } + + public override Id AssemblyPrefix + { + get + { + var ct = ContainingType; + return ct != null ? ct.AssemblyPrefix : IsPrimitiveType ? builtin : cx.AssemblyPrefix; + } + } + + IEnumerable MakeTypeParameters() + { + if (ThisTypeParameters == 0) + return Enumerable.Empty(); + + var typeParams = new TypeTypeParameter[ThisTypeParameters]; + var genericParams = td.GetGenericParameters(); + int toSkip = genericParams.Count - typeParams.Length; + + // Two-phase population because type parameters can be mutually dependent + for (int i = 0; i < typeParams.Length; ++i) + typeParams[i] = cx.Populate(new TypeTypeParameter(this, this, i)); + for (int i = 0; i < typeParams.Length; ++i) + typeParams[i].PopulateHandle(this, genericParams[i + toSkip]); + return typeParams; + } + + readonly Lazy> typeParams; + + public override IEnumerable MethodParameters => Enumerable.Empty(); + + public override IEnumerable TypeParameters + { + get + { + if (declType != null) + { + foreach (var t in declType.TypeParameters) + yield return t; + } + + foreach (var t in typeParams.Value) + yield return t; + } + } + + public override IEnumerable Contents + { + get + { + foreach (var c in base.Contents) yield return c; + + MakeTypeParameters(); + + foreach (var f in td.GetFields()) + { + // Populate field if needed + yield return cx.CreateGeneric(this, f); + } + + foreach (var prop in td.GetProperties()) + { + yield return new Property(this, this, prop); + } + + foreach (var @event in td.GetEvents()) + { + yield return new Event(cx, this, @event); + } + + foreach (var a in Attribute.Populate(cx, this, td.GetCustomAttributes())) + yield return a; + + foreach (var impl in td.GetMethodImplementations().Select(i => cx.mdReader.GetMethodImplementation(i))) + { + var m = (Method)cx.CreateGeneric(this, impl.MethodBody); + var decl = (Method)cx.CreateGeneric(this, impl.MethodDeclaration); + + yield return m; + yield return decl; + yield return Tuples.cil_implements(m, decl); + } + + if (td.Attributes.HasFlag(TypeAttributes.Abstract)) + yield return Tuples.cil_abstract(this); + + if (td.Attributes.HasFlag(TypeAttributes.Class)) + yield return Tuples.cil_class(this); + + if (td.Attributes.HasFlag(TypeAttributes.Interface)) + yield return Tuples.cil_interface(this); + + if (td.Attributes.HasFlag(TypeAttributes.Public)) + yield return Tuples.cil_public(this); + + if (td.Attributes.HasFlag(TypeAttributes.Sealed)) + yield return Tuples.cil_sealed(this); + + if (td.Attributes.HasFlag(TypeAttributes.HasSecurity)) + yield return Tuples.cil_security(this); + + // Base types + + if (!td.BaseType.IsNil) + { + var @base = (Type)cx.CreateGeneric(this, td.BaseType); + yield return @base; + yield return Tuples.cil_base_class(this, @base); + } + + foreach (var @interface in td.GetInterfaceImplementations().Select(i => cx.mdReader.GetInterfaceImplementation(i))) + { + var t = (Type)cx.CreateGeneric(this, @interface.Interface); + yield return t; + yield return Tuples.cil_base_interface(this, t); + } + + // Only type definitions have locations. + yield return Tuples.cil_type_location(this, cx.assembly); + } + } + + internal override Method LookupMethod(StringHandle name, BlobHandle signature) + { + foreach (var h in td.GetMethods()) + { + var md = cx.mdReader.GetMethodDefinition(h); + + if (md.Name == name && md.Signature == signature) + { + return (Method)cx.Create(h); + } + } + + throw new InternalError("Couldn't locate method in type"); + } + } + + /// + /// A type reference, to a type in a referenced assembly. + /// + public sealed class TypeReferenceType : Type + { + readonly TypeReference tr; + readonly Lazy typeParams; + + public TypeReferenceType(Context cx, TypeReferenceHandle handle) : this(cx, cx.mdReader.GetTypeReference(handle)) + { + ShortId = MakeId(false); + typeParams = new Lazy(MakeTypeParameters); + } + + public TypeReferenceType(Context cx, TypeReference tr) : base(cx) + { + this.tr = tr; + } + + TypeTypeParameter[] MakeTypeParameters() + { + var typeParams = new TypeTypeParameter[ThisTypeParameters]; + for (int i = 0; i < typeParams.Length; ++i) + { + typeParams[i] = new TypeTypeParameter(this, this, i); + } + return typeParams; + } + + public override IEnumerable Contents + { + get + { + foreach (var tp in typeParams.Value) + yield return tp; + + foreach (var c in base.Contents) + yield return c; + } + } + + public override Id Name + { + get + { + var name = cx.GetId(tr.Name); + var tick = name.Value.IndexOf('`'); + return tick == -1 ? name : cx.GetId(name.Value.Substring(0, tick)); + } + } + + public override Namespace Namespace => cx.CreateNamespace(tr.Namespace); + + public override int ThisTypeParameters + { + get + { + // Parse the name + var name = cx.GetId(tr.Name); + var tick = name.Value.IndexOf('`'); + return tick == -1 ? 0 : int.Parse(name.Value.Substring(tick + 1)); + } + } + + public override IEnumerable ThisGenericArguments + { + get + { + foreach (var t in typeParams.Value) + yield return t; + } + } + + public override Type ContainingType + { + get + { + if (tr.ResolutionScope.Kind == HandleKind.TypeReference) + return (Type)cx.Create((TypeReferenceHandle)tr.ResolutionScope); + return null; + } + } + + public override CilTypeKind Kind => CilTypeKind.ValueOrRefType; + + public override Id AssemblyPrefix + { + get + { + switch (tr.ResolutionScope.Kind) + { + case HandleKind.TypeReference: + return ContainingType.AssemblyPrefix; + case HandleKind.AssemblyReference: + var assemblyDef = cx.mdReader.GetAssemblyReference((AssemblyReferenceHandle)tr.ResolutionScope); + return cx.GetId(assemblyDef.Name) + "_" + cx.GetId(assemblyDef.Version.ToString()) + "::"; + default: + return cx.AssemblyPrefix; + } + } + } + + public override IEnumerable TypeParameters => typeParams.Value; + + public override IEnumerable MethodParameters => throw new InternalError("This type does not have method parameters"); + + public override Id MakeId(bool inContext) + { + if (IsPrimitiveType) return PrimitiveTypeId; + + var ct = ContainingType; + Id l = null; + if (ct != null) + { + l = ContainingType.GetId(inContext); + } + else + { + if (tr.ResolutionScope.Kind == HandleKind.AssemblyReference) + { + l = AssemblyPrefix; + } + + if (!Namespace.IsGlobalNamespace) + { + l += Namespace.ShortId; + } + } + + return l + cx.Dot + cx.GetId(tr.Name); + } + + public override Type Construct(IEnumerable typeArguments) + { + if (TotalTypeParametersCheck != typeArguments.Count()) + throw new InternalError("Mismatched type arguments"); + + return cx.Populate(new ConstructedType(cx, this, typeArguments)); + } + } + + + /// + /// A constructed type. + /// + public sealed class ConstructedType : Type + { + readonly Type unboundGenericType; + readonly Type[] thisTypeArguments; + + public override IEnumerable ThisTypeArguments => thisTypeArguments; + + public override IEnumerable ThisGenericArguments => thisTypeArguments.EnumerateNull(); + + public override IEnumerable Contents + { + get + { + foreach (var c in base.Contents) + yield return c; + + int i = 0; + foreach (var type in ThisGenericArguments) + { + yield return type; + yield return Tuples.cil_type_argument(this, i++, type); + } + } + } + + public override Type SourceDeclaration => unboundGenericType; + + public ConstructedType(Context cx, Type unboundType, IEnumerable typeArguments) : base(cx) + { + var suppliedArgs = typeArguments.Count(); + if (suppliedArgs != unboundType.TotalTypeParametersCheck) + throw new InternalError("Unexpected number of type arguments in ConstructedType"); + + unboundGenericType = unboundType; + var thisParams = unboundType.ThisTypeParameters; + var parentParams = suppliedArgs - thisParams; + + if (typeArguments.Count() == thisParams) + { + containingType = unboundType.ContainingType; + thisTypeArguments = typeArguments.ToArray(); + } + else if (thisParams == 0) + { + containingType = unboundType.ContainingType.Construct(typeArguments); + } + else + { + containingType = unboundType.ContainingType.Construct(typeArguments.Take(parentParams)); + thisTypeArguments = typeArguments.Skip(parentParams).ToArray(); + } + + ShortId = MakeId(false); + } + + readonly Type containingType; + public override Type ContainingType => containingType; + + public override Id Name => unboundGenericType.Name; + + public override Namespace Namespace => unboundGenericType.Namespace; + + public override int ThisTypeParameters => thisTypeArguments == null ? 0 : thisTypeArguments.Length; + + public override CilTypeKind Kind => unboundGenericType.Kind; + + public override Type Construct(IEnumerable typeArguments) + { + throw new NotImplementedException(); + } + + public override Id MakeId(bool inContext) + { + Id l; + if (ContainingType != null) + { + l = ContainingType.GetId(inContext) + cx.Dot; + } + else + { + l = AssemblyPrefix; + + if (!Namespace.IsGlobalNamespace) + { + l += Namespace.ShortId + cx.Dot; + } + } + l += unboundGenericType.Name; + + if (thisTypeArguments != null && thisTypeArguments.Any()) + { + l += open; + bool first = true; + foreach (var t in thisTypeArguments) + { + if (first) first = false; else l += comma; + l += t.ShortId; + } + l += close; + } + return l; + } + + static readonly StringId open = new StringId("<"); + static readonly StringId close = new StringId(">"); + static readonly StringId comma = new StringId(","); + + public override Id AssemblyPrefix => unboundGenericType.AssemblyPrefix; + + public override IEnumerable TypeParameters => GenericArguments; + + public override IEnumerable MethodParameters => throw new NotImplementedException(); + } + + public sealed class PrimitiveType : Type + { + readonly PrimitiveTypeCode typeCode; + public PrimitiveType(Context cx, PrimitiveTypeCode tc) : base(cx) + { + typeCode = tc; + ShortId = MakeId(false); + } + + public override Id MakeId(bool inContext) + { + return builtin + Name; + } + + public override Id Name => typeCode.Id(); + + public override Namespace Namespace => cx.SystemNamespace; + + public override Type ContainingType => null; + + public override int ThisTypeParameters => 0; + + public override CilTypeKind Kind => CilTypeKind.ValueOrRefType; + + static readonly Id empty = new StringId(""); + + public override Id AssemblyPrefix => empty; + + public override IEnumerable TypeParameters => throw new NotImplementedException(); + + public override IEnumerable MethodParameters => throw new NotImplementedException(); + + public override Type Construct(IEnumerable typeArguments) => throw new NotImplementedException(); + } + + /// + /// An array type. + /// + sealed class ArrayType : Type, IArrayType + { + readonly Type elementType; + readonly int rank; + + public ArrayType(Context cx, Type element, ArrayShape shape) : base(cx) + { + rank = shape.Rank; + elementType = element; + ShortId = MakeId(false); + } + + public ArrayType(Context cx, Type element) : base(cx) + { + rank = 1; + elementType = element; + ShortId = MakeId(false); + } + + public override Id MakeId(bool inContext) => elementType.GetId(inContext) + openBracket + rank + closeBracket; + + static readonly StringId openBracket = new StringId("[]"); + static readonly StringId closeBracket = new StringId("[]"); + + public override Id Name => elementType.Name + openBracket + closeBracket; + + public override Namespace Namespace => cx.SystemNamespace; + + public override Type ContainingType => null; + + public override int ThisTypeParameters => elementType.ThisTypeParameters; + + public override CilTypeKind Kind => CilTypeKind.Array; + + public override Type Construct(IEnumerable typeArguments) => cx.Populate(new ArrayType(cx, elementType.Construct(typeArguments))); + + public override Type SourceDeclaration => cx.Populate(new ArrayType(cx, elementType.SourceDeclaration)); + + public override IEnumerable Contents + { + get + { + foreach (var c in base.Contents) + yield return c; + + yield return Tuples.cil_array_type(this, elementType, rank); + } + } + + public override Id AssemblyPrefix + { + get + { + return elementType.AssemblyPrefix; + } + } + + public override IEnumerable GenericArguments => elementType.GenericArguments; + + public override IEnumerable TypeParameters => elementType.TypeParameters; + + public override IEnumerable MethodParameters => throw new NotImplementedException(); + } + + interface ITypeParameter : IType + { + } + + abstract class TypeParameter : Type, ITypeParameter + { + protected readonly GenericContext gc; + + public TypeParameter(GenericContext gc) : base(gc.cx) + { + this.gc = gc; + } + + public override Namespace Namespace => null; + + public override Type ContainingType => null; + + public override int ThisTypeParameters => 0; + + public override CilTypeKind Kind => CilTypeKind.TypeParameter; + + public override Id AssemblyPrefix => throw new NotImplementedException(); + + public override Type Construct(IEnumerable typeArguments) => throw new InternalError("Attempt to construct a type parameter"); + + public IEnumerable PopulateHandle(GenericContext gc, GenericParameterHandle parameterHandle) + { + if (!parameterHandle.IsNil) + { + var tp = cx.mdReader.GetGenericParameter(parameterHandle); + + if (tp.Attributes.HasFlag(GenericParameterAttributes.Contravariant)) + yield return Tuples.cil_typeparam_contravariant(this); + if (tp.Attributes.HasFlag(GenericParameterAttributes.Covariant)) + yield return Tuples.cil_typeparam_covariant(this); + if (tp.Attributes.HasFlag(GenericParameterAttributes.DefaultConstructorConstraint)) + yield return Tuples.cil_typeparam_new(this); + if (tp.Attributes.HasFlag(GenericParameterAttributes.ReferenceTypeConstraint)) + yield return Tuples.cil_typeparam_class(this); + if (tp.Attributes.HasFlag(GenericParameterAttributes.NotNullableValueTypeConstraint)) + yield return Tuples.cil_typeparam_struct(this); + + foreach (var constraint in tp.GetConstraints().Select(h => cx.mdReader.GetGenericParameterConstraint(h))) + { + var t = (Type)cx.CreateGeneric(this.gc, constraint.Type); + yield return t; + yield return Tuples.cil_typeparam_constraint(this, t); + } + } + } + } + + sealed class MethodTypeParameter : TypeParameter + { + readonly Method method; + readonly int index; + + public override Id MakeId(bool inContext) => inContext && method == gc ? Name : method.ShortId + Name; + + static readonly Id excl = new StringId("!"); + + public override Id Name => excl + index.ToString(); + + public MethodTypeParameter(GenericContext gc, Method m, int index) : base(gc) + { + method = m; + this.index = index; + ShortId = MakeId(false); + } + + public override TypeContainer Parent => method; + + public override IEnumerable TypeParameters => throw new NotImplementedException(); + + public override IEnumerable MethodParameters => throw new NotImplementedException(); + + public override IEnumerable Contents + { + get + { + yield return Tuples.cil_type(this, Name.Value, Kind, method, SourceDeclaration); + yield return Tuples.cil_type_parameter(method, index, this); + } + } + } + + + sealed class TypeTypeParameter : TypeParameter + { + readonly Type type; + readonly int index; + + public TypeTypeParameter(GenericContext cx, Type t, int i) : base(cx) + { + index = i; + type = t; + ShortId = t.ShortId + Name; + } + + public override Id MakeId(bool inContext) => type.MakeId(inContext) + Name; + + public override TypeContainer Parent => type ?? gc as TypeContainer; + + static readonly Id excl = new StringId("!"); + public override Id Name => excl + index.ToString(); + + public override IEnumerable TypeParameters => Enumerable.Empty(); + + public override IEnumerable MethodParameters => Enumerable.Empty(); + + public override IEnumerable Contents + { + get + { + yield return Tuples.cil_type(this, Name.Value, Kind, type, SourceDeclaration); + yield return Tuples.cil_type_parameter(type, index, this); + } + } + } + + interface IPointerType : IType + { + } + + sealed class PointerType : Type, IPointerType + { + readonly Type pointee; + + public PointerType(Context cx, Type pointee) : base(cx) + { + this.pointee = pointee; + ShortId = MakeId(false); + } + + public override Id MakeId(bool inContext) => pointee.MakeId(inContext) + star; + + static readonly StringId star = new StringId("*"); + + public override Id Name => pointee.Name + star; + + public override Namespace Namespace => pointee.Namespace; + + public override Type ContainingType => pointee.ContainingType; + + public override TypeContainer Parent => pointee.Parent; + + public override int ThisTypeParameters => 0; + + public override CilTypeKind Kind => CilTypeKind.Pointer; + + public override Id AssemblyPrefix => pointee.AssemblyPrefix; + + public override IEnumerable TypeParameters => throw new NotImplementedException(); + + public override IEnumerable MethodParameters => throw new NotImplementedException(); + + public override Type Construct(IEnumerable typeArguments) => throw new NotImplementedException(); + + public override IEnumerable Contents + { + get + { + foreach (var c in base.Contents) yield return c; + yield return Tuples.cil_pointer_type(this, pointee); + } + } + } + + sealed class ErrorType : Type + { + public ErrorType(Context cx) : base(cx) + { + ShortId = MakeId(false); + } + + public override Id MakeId(bool inContext) => CIL.Id.Create(""); + + public override CilTypeKind Kind => CilTypeKind.ValueOrRefType; + + public override Id Name => new StringId("!error"); + + public override Namespace Namespace => cx.GlobalNamespace; + + public override Type ContainingType => null; + + public override int ThisTypeParameters => 0; + + public override Id AssemblyPrefix => throw new NotImplementedException(); + + public override IEnumerable TypeParameters => throw new NotImplementedException(); + + public override IEnumerable MethodParameters => throw new NotImplementedException(); + + public override Type Construct(IEnumerable typeArguments) => throw new NotImplementedException(); + } + + public sealed class TypeSpecificationType : Type + { + readonly TypeSpecification ts; + readonly Type decodedType; + + public TypeSpecificationType(GenericContext gc, TypeSpecificationHandle handle) : base(gc.cx) + { + ts = cx.mdReader.GetTypeSpecification(handle); + decodedType = ts.DecodeSignature(cx.TypeSignatureDecoder, gc); + ShortId = decodedType.ShortId; + } + + public override Id MakeId(bool inContext) => decodedType.MakeId(inContext); + + public override IEnumerable Contents + { + get + { + yield return decodedType; + } + } + + public override Id AssemblyPrefix => throw new NotImplementedException(); + + public override CilTypeKind Kind => throw new NotImplementedException(); + + public override Id Name => decodedType.Name; + + public override Namespace Namespace => throw new NotImplementedException(); + + public override Type ContainingType => decodedType.ContainingType; + + public override int ThisTypeParameters => throw new NotImplementedException(); + + public override IEnumerable TypeParameters => decodedType.TypeParameters; + + public override IEnumerable MethodParameters => throw new NotImplementedException(); + + public override Type Construct(IEnumerable typeArguments) => throw new NotImplementedException(); + + public override Type SourceDeclaration => decodedType.SourceDeclaration; + } + + interface ITypeSignature + { + Id MakeId(GenericContext gc); + } + + public class SignatureDecoder : ISignatureTypeProvider + { + struct Array : ITypeSignature + { + public ITypeSignature elementType; + public ArrayShape shape; + public Id MakeId(GenericContext gc) => elementType.MakeId(gc) + "[]"; // Make these static + } + + struct ByRef : ITypeSignature + { + public ITypeSignature elementType; + public Id MakeId(GenericContext gc) => "ref " + elementType.MakeId(gc); + } + + struct FnPtr : ITypeSignature + { + public MethodSignature signature; + public Id MakeId(GenericContext gc) => Id.Create(""); // !! + } + + ITypeSignature IConstructedTypeProvider.GetArrayType(ITypeSignature elementType, ArrayShape shape) => + new Array { elementType = elementType, shape = shape }; + + ITypeSignature IConstructedTypeProvider.GetByReferenceType(ITypeSignature elementType) => + new ByRef { elementType = elementType }; + + ITypeSignature ISignatureTypeProvider.GetFunctionPointerType(MethodSignature signature) => + new FnPtr { signature = signature }; + + class Instantiation : ITypeSignature + { + public ITypeSignature genericType; + public ImmutableArray typeArguments; + public Id MakeId(GenericContext gc) => + genericType.MakeId(gc) + "<" + Id.CommaSeparatedList(typeArguments.Select(arg => arg.MakeId(gc))) + ">"; + } + + ITypeSignature IConstructedTypeProvider.GetGenericInstantiation(ITypeSignature genericType, ImmutableArray typeArguments) => + new Instantiation { genericType = genericType, typeArguments = typeArguments }; + + class GenericMethodParameter : ITypeSignature + { + public int index; + static readonly Id excl = Id.Create("M!"); + public Id MakeId(GenericContext gc) => excl + index; + } + + class GenericTypeParameter : ITypeSignature + { + public int index; + static readonly Id excl = Id.Create("T!"); + public Id MakeId(GenericContext gc) => excl + index; + } + + ITypeSignature ISignatureTypeProvider.GetGenericMethodParameter(object genericContext, int index) => + new GenericMethodParameter { index = index }; + + ITypeSignature ISignatureTypeProvider.GetGenericTypeParameter(object genericContext, int index) => + new GenericTypeParameter { index = index }; + + class Modified : ITypeSignature + { + public ITypeSignature modifier; + public ITypeSignature unmodifiedType; + public bool isRequired; + + public Id MakeId(GenericContext gc) => unmodifiedType.MakeId(gc); + } + + ITypeSignature ISignatureTypeProvider.GetModifiedType(ITypeSignature modifier, ITypeSignature unmodifiedType, bool isRequired) + { + return new Modified { modifier = modifier, unmodifiedType = unmodifiedType, isRequired = isRequired }; + } + + class Pinned : ITypeSignature + { + public ITypeSignature elementType; + public Id MakeId(GenericContext gc) => "pinned " + elementType.MakeId(gc); + } + + ITypeSignature ISignatureTypeProvider.GetPinnedType(ITypeSignature elementType) + { + return new Pinned { elementType = elementType }; + } + + class PointerType : ITypeSignature + { + public ITypeSignature elementType; + public Id MakeId(GenericContext gc) => elementType.MakeId(gc) + "*"; + } + + ITypeSignature IConstructedTypeProvider.GetPointerType(ITypeSignature elementType) + { + return new PointerType { elementType = elementType }; + } + + class Primitive : ITypeSignature + { + public PrimitiveTypeCode typeCode; + public Id MakeId(GenericContext gc) => typeCode.Id(); + } + + ITypeSignature ISimpleTypeProvider.GetPrimitiveType(PrimitiveTypeCode typeCode) + { + return new Primitive { typeCode = typeCode }; + } + + class SzArrayType : ITypeSignature + { + public ITypeSignature elementType; + public Id MakeId(GenericContext gc) => elementType.MakeId(gc) + "[]"; + } + + ITypeSignature ISZArrayTypeProvider.GetSZArrayType(ITypeSignature elementType) + { + return new SzArrayType { elementType = elementType }; + } + + class TypeDefinition : ITypeSignature + { + public TypeDefinitionHandle handle; + public byte rawTypeKind; + Type type; + public Id MakeId(GenericContext gc) + { + type = (Type)gc.cx.Create(handle); + return type.ShortId; + } + } + + ITypeSignature ISimpleTypeProvider.GetTypeFromDefinition(MetadataReader reader, TypeDefinitionHandle handle, byte rawTypeKind) + { + return new TypeDefinition { handle = handle, rawTypeKind = rawTypeKind }; + } + + class TypeReference : ITypeSignature + { + public TypeReferenceHandle handle; + public byte rawTypeKind; // struct/class (not used) + Type type; + public Id MakeId(GenericContext gc) + { + type = (Type)gc.cx.Create(handle); + return type.ShortId; + } + } + + ITypeSignature ISimpleTypeProvider.GetTypeFromReference(MetadataReader reader, TypeReferenceHandle handle, byte rawTypeKind) + { + return new TypeReference { handle = handle, rawTypeKind = rawTypeKind }; + } + + ITypeSignature ISignatureTypeProvider.GetTypeFromSpecification(MetadataReader reader, object genericContext, TypeSpecificationHandle handle, byte rawTypeKind) + { + var ts = reader.GetTypeSpecification(handle); + return ts.DecodeSignature(this, genericContext); + } + } + + + /// + /// Decodes a type signature and produces a Type, for use by DecodeSignature() and friends. + /// + public class TypeSignatureDecoder : ISignatureTypeProvider + { + readonly Context cx; + + public TypeSignatureDecoder(Context cx) + { + this.cx = cx; + } + + Type IConstructedTypeProvider.GetArrayType(Type elementType, ArrayShape shape) => + cx.Populate(new ArrayType(cx, elementType, shape)); + + Type IConstructedTypeProvider.GetByReferenceType(Type elementType) => + elementType; // ?? + + Type ISignatureTypeProvider.GetFunctionPointerType(MethodSignature signature) => + cx.ErrorType; // Don't know what to do !! + + Type IConstructedTypeProvider.GetGenericInstantiation(Type genericType, ImmutableArray typeArguments) => + genericType.Construct(typeArguments); + + Type ISignatureTypeProvider.GetGenericMethodParameter(GenericContext genericContext, int index) => + genericContext.GetGenericMethodParameter(index); + + Type ISignatureTypeProvider.GetGenericTypeParameter(GenericContext genericContext, int index) => + genericContext.GetGenericTypeParameter(index); + + Type ISignatureTypeProvider.GetModifiedType(Type modifier, Type unmodifiedType, bool isRequired) => + // !! Not implemented properly + unmodifiedType; + + Type ISignatureTypeProvider.GetPinnedType(Type elementType) => + cx.Populate(new PointerType(cx, elementType)); + + Type IConstructedTypeProvider.GetPointerType(Type elementType) => + cx.Populate(new PointerType(cx, elementType)); + + Type ISimpleTypeProvider.GetPrimitiveType(PrimitiveTypeCode typeCode) => + cx.Populate(new PrimitiveType(cx, typeCode)); + + Type ISZArrayTypeProvider.GetSZArrayType(Type elementType) => + cx.Populate(new ArrayType(cx, elementType)); + + Type ISimpleTypeProvider.GetTypeFromDefinition(MetadataReader reader, TypeDefinitionHandle handle, byte rawTypeKind) => + (Type)cx.Create(handle); + + Type ISimpleTypeProvider.GetTypeFromReference(MetadataReader reader, TypeReferenceHandle handle, byte rawTypeKind) => + (Type)cx.Create(handle); + + Type ISignatureTypeProvider.GetTypeFromSpecification(MetadataReader reader, GenericContext genericContext, TypeSpecificationHandle handle, byte rawTypeKind) => + throw new NotImplementedException(); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/ExtractionProduct.cs b/csharp/extractor/Semmle.Extraction.CIL/ExtractionProduct.cs new file mode 100644 index 00000000000..16181d9b32e --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/ExtractionProduct.cs @@ -0,0 +1,127 @@ +using System; +using System.Collections.Generic; + +namespace Semmle.Extraction.CIL +{ + /// + /// Something that is extracted from an entity. + /// + /// + /// + /// The extraction algorithm proceeds as follows: + /// - Construct entity + /// - Call Extract() + /// - ILabelledEntity check if already extracted + /// - Enumerate Contents to produce more extraction products + /// - Extract these until there is nothing left to extract + /// + public interface IExtractionProduct + { + /// + /// Perform further extraction/population of this item as necessary. + /// + /// + /// The extraction context. + void Extract(Context cx); + } + + /// + /// An entity which has been extracted. + /// + public interface IExtractedEntity : IEntity, IExtractionProduct + { + /// + /// The contents of the entity. + /// + IEnumerable Contents { get; } + } + + /// + /// An entity that has contents to extract. There is no need to populate + /// a key as it's done in the contructor. + /// + public abstract class UnlabelledEntity : IExtractedEntity + { + public abstract IEnumerable Contents { get; } + public Label Label { get; set; } + + public Microsoft.CodeAnalysis.Location ReportingLocation => throw new NotImplementedException(); + + public virtual IId Id => FreshId.Instance; + + public virtual void Extract(Context cx) + { + cx.Extract(this); + } + + public readonly Context cx; + + protected UnlabelledEntity(Context cx) + { + this.cx = cx; + cx.cx.AddFreshLabel(this); + } + + TrapStackBehaviour IEntity.TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } + + /// + /// An entity that needs to be populated during extraction. + /// This assigns a key and optionally extracts its contents. + /// + public abstract class LabelledEntity : ILabelledEntity + { + public abstract IEnumerable Contents { get; } + public Label Label { get; set; } + public Microsoft.CodeAnalysis.Location ReportingLocation => throw new NotImplementedException(); + + public Id ShortId { get; set; } + public abstract Id IdSuffix { get; } + public IId Id => ShortId + IdSuffix; + + public void Extract(Context cx) + { + cx.Populate(this); + } + + public readonly Context cx; + + protected LabelledEntity(Context cx) + { + this.cx = cx; + } + + public override string ToString() => Id.ToString(); + + TrapStackBehaviour IEntity.TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } + + /// + /// An entity with a defined ID. + /// + public interface ILabelledEntity : IExtractedEntity + { + Id ShortId { get; set; } + Id IdSuffix { get; } + } + + /// + /// A tuple that is an extraction product. + /// + class Tuple : IExtractionProduct + { + readonly Extraction.Tuple tuple; + + public Tuple(string name, params object[] args) + { + tuple = new Extraction.Tuple(name, args); + } + + public void Extract(Context cx) + { + cx.cx.Emit(tuple); + } + + public override string ToString() => tuple.ToString(); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Factories.cs b/csharp/extractor/Semmle.Extraction.CIL/Factories.cs new file mode 100644 index 00000000000..b8909f2b833 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Factories.cs @@ -0,0 +1,233 @@ +using System; +using System.Collections.Generic; +using System.Reflection.Metadata; + +namespace Semmle.Extraction.CIL +{ + /// + /// Provides methods for creating and caching various entities. + /// + public partial class Context + { + readonly Dictionary ids = new Dictionary(); + + public T Populate(T e) where T : ILabelledEntity + { + Id id = e.ShortId; + + if (ids.TryGetValue(id, out var existing)) + { + // It exists already + e.Label = existing.Item1; + e.ShortId = existing.Item2; // Reuse ID for efficiency + } + else + { + cx.DefineLabel(e); + ids.Add(id, (e.Label, id)); + cx.PopulateLater(() => + { + foreach (var c in e.Contents) + c.Extract(this); + }); + } + return e; + } + + public IExtractedEntity Create(Handle h) + { + var entity = CreateGeneric(defaultGenericContext, h); + return entity; + } + + /// + /// Creates an entity from a Handle in a GenericContext. + /// The type of the returned entity depends on the type of the handle. + /// The GenericContext is needed because some handles are generics which + /// need to be expanded in terms of the current instantiation. If this sounds + /// complex, you are right. + /// + /// The pair (h,genericContext) is cached in case it is needed again. + /// + /// The handle of the entity. + /// The generic context. + /// + public ILabelledEntity CreateGeneric(GenericContext genericContext, Handle h) => genericHandleFactory[genericContext, h]; + + readonly GenericContext defaultGenericContext; + + ILabelledEntity CreateGenericHandle(GenericContext gc, Handle handle) + { + ILabelledEntity entity; + switch (handle.Kind) + { + case HandleKind.MethodDefinition: + entity = new Entities.DefinitionMethod(gc, (MethodDefinitionHandle)handle); + break; + case HandleKind.MemberReference: + entity = Create(gc, (MemberReferenceHandle)handle); + break; + case HandleKind.MethodSpecification: + entity = new Entities.MethodSpecificationMethod(gc, (MethodSpecificationHandle)handle); + break; + case HandleKind.FieldDefinition: + entity = new Entities.DefinitionField(gc, (FieldDefinitionHandle)handle); + break; + case HandleKind.TypeReference: + entity = new Entities.TypeReferenceType(this, (TypeReferenceHandle)handle); + break; + case HandleKind.TypeSpecification: + entity = new Entities.TypeSpecificationType(gc, (TypeSpecificationHandle)handle); + break; + case HandleKind.TypeDefinition: + entity = new Entities.TypeDefinitionType(this, (TypeDefinitionHandle)handle); + break; + default: + throw new InternalError("Unhandled handle kind " + handle.Kind); + } + + Populate(entity); + return entity; + } + + ILabelledEntity Create(GenericContext gc, MemberReferenceHandle handle) + { + var mr = mdReader.GetMemberReference(handle); + switch (mr.GetKind()) + { + case MemberReferenceKind.Method: + return new Entities.MemberReferenceMethod(gc, handle); + case MemberReferenceKind.Field: + return new Entities.MemberReferenceField(gc, handle); + default: + throw new InternalError("Unhandled member reference handle"); + } + } + + #region Strings + readonly Dictionary stringHandleIds = new Dictionary(); + readonly Dictionary stringIds = new Dictionary(); + + /// + /// Return an ID containing the given string. + /// + /// The string handle. + /// An ID. + public StringId GetId(StringHandle h) + { + StringId result; + if (!stringHandleIds.TryGetValue(h, out result)) + { + result = new StringId(mdReader.GetString(h)); + stringHandleIds.Add(h, result); + } + return result; + } + + public readonly StringId Dot = new StringId("."); + + /// + /// Gets an ID containing the given string. + /// Caches existing IDs for more compact storage. + /// + /// The string. + /// An ID containing the string. + public StringId GetId(string str) + { + StringId result; + if (!stringIds.TryGetValue(str, out result)) + { + result = new StringId(str); + stringIds.Add(str, result); + } + return result; + } + #endregion + + #region Namespaces + + readonly CachedFunction namespaceFactory; + + public Entities.Namespace CreateNamespace(StringHandle fqn) => namespaceFactory[fqn]; + + readonly Lazy globalNamespace, systemNamespace; + + /// + /// The entity representing the global namespace. + /// + public Entities.Namespace GlobalNamespace => globalNamespace.Value; + + /// + /// The entity representing the System namespace. + /// + public Entities.Namespace SystemNamespace => systemNamespace.Value; + + /// + /// Creates a namespace from a fully-qualified name. + /// + /// The fully-qualified namespace name. + /// The namespace entity. + Entities.Namespace CreateNamespace(string fqn) => Populate(new Entities.Namespace(this, fqn)); + + readonly CachedFunction namespaceDefinitionFactory; + + /// + /// Creates a namespace from a namespace handle. + /// + /// The handle of the namespace. + /// The namespace entity. + public Entities.Namespace Create(NamespaceDefinitionHandle handle) => namespaceDefinitionFactory[handle]; + + Entities.Namespace CreateNamespace(NamespaceDefinitionHandle handle) + { + if (handle.IsNil) return GlobalNamespace; + NamespaceDefinition nd = mdReader.GetNamespaceDefinition(handle); + return Populate(new Entities.Namespace(this, GetId(nd.Name), Create(nd.Parent))); + } + #endregion + + #region Locations + readonly CachedFunction sourceFiles; + readonly CachedFunction folders; + readonly CachedFunction sourceLocations; + + /// + /// Creates a source file entity from a PDB source file. + /// + /// The PDB source file. + /// A source file entity. + public Entities.PdbSourceFile CreateSourceFile(PDB.ISourceFile file) => sourceFiles[file]; + + /// + /// Creates a folder entitiy with the given path. + /// + /// The path of the folder. + /// A folder entity. + public Entities.Folder CreateFolder(string path) => folders[path]; + + /// + /// Creates a source location. + /// + /// The source location from PDB. + /// A source location entity. + public Entities.PdbSourceLocation CreateSourceLocation(PDB.Location loc) => sourceLocations[loc]; + + #endregion + + readonly CachedFunction genericHandleFactory; + + /// + /// Gets the short name of a member, without the preceding interface qualifier. + /// + /// The handle of the name. + /// The short name. + public string ShortName(StringHandle handle) + { + string str = mdReader.GetString(handle); + if (str.EndsWith(".ctor")) return ".ctor"; + if (str.EndsWith(".cctor")) return ".cctor"; + var dot = str.LastIndexOf('.'); + return dot == -1 ? str : str.Substring(dot + 1); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Id.cs b/csharp/extractor/Semmle.Extraction.CIL/Id.cs new file mode 100644 index 00000000000..f55d8acc9cd --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Id.cs @@ -0,0 +1,204 @@ +using System.Collections.Generic; +using System.Reflection.Metadata; + +namespace Semmle.Extraction.CIL +{ + /// + /// An ID fragment which is designed to be shared, reused + /// and composed using the + operator. + /// + public abstract class Id : IId + { + public void AppendTo(ITrapBuilder tb) + { + tb.Append("@\""); + BuildParts(tb); + tb.Append("\""); + } + + public abstract void BuildParts(ITrapBuilder tb); + + public static Id operator +(Id l1, Id l2) => Create(l1, l2); + + public static Id operator +(Id l1, string l2) => Create(l1, Create(l2)); + + public static Id operator +(Id l1, int l2) => Create(l1, Create(l2)); + + public static Id operator +(string l1, Id l2) => Create(Create(l1), l2); + + public static Id operator +(int l1, Id l2) => Create(Create(l1), l2); + + public static Id Create(string s) => s == null ? null : new StringId(s); + + public static Id Create(int i) => new IntId(i); + + static readonly Id openCurly = Create("{#"); + static readonly Id closeCurly = Create("}"); + + public static Id Create(Label l) => openCurly + l.Value + closeCurly; + + static readonly Id comma = Id.Create(","); + + public static Id CommaSeparatedList(IEnumerable items) + { + Id result = null; + bool first = true; + foreach (var i in items) + { + if (first) first = false; else result += comma; + result += i; + } + return result; + } + + public static Id Create(Id l1, Id l2) + { + return l1 == null ? l2 : l2 == null ? l1 : new ConsId(l1, l2); + } + + public abstract string Value { get; } + + public override string ToString() => Value; + } + + /// + /// An ID concatenating two other IDs. + /// + public sealed class ConsId : Id + { + readonly Id left, right; + readonly int hash; + + public ConsId(Id l1, Id l2) + { + left = l1; + right = l2; + hash = unchecked(12 + 3 * (left.GetHashCode() + 51 * right.GetHashCode())); + } + + public override void BuildParts(ITrapBuilder tb) + { + left.BuildParts(tb); + right.BuildParts(tb); + } + + public override bool Equals(object other) + { + return other is ConsId && Equals((ConsId)other); + } + + public bool Equals(ConsId other) + { + return this == other || + (hash == other.hash && left.Equals(other.left) && right.Equals(other.right)); + } + + public override int GetHashCode() => hash; + + public override string Value => left.Value + right.Value; + + } + + /// + /// A leaf ID storing a string. + /// + public sealed class StringId : Id + { + readonly string value; + public override string Value => value; + + public StringId(string s) + { + value = s; + } + + public override void BuildParts(ITrapBuilder tb) + { + tb.Append(value); + } + + public override bool Equals(object obj) + { + return obj is StringId && ((StringId)obj).value == value; + } + + public override int GetHashCode() => Value.GetHashCode() * 31 + 9; + } + + /// + /// A leaf ID storing an integer. + /// + public sealed class IntId : Id + { + readonly int value; + public override string Value => value.ToString(); + + public IntId(int i) + { + value = i; + } + + public override void BuildParts(ITrapBuilder tb) + { + tb.Append(value); + } + + public override bool Equals(object obj) + { + return obj is IntId && ((IntId)obj).value == value; + } + + public override int GetHashCode() => unchecked(12 + value * 17); + } + + /// + /// Some predefined IDs. + /// + public static class IdUtils + { + public static StringId boolId = new StringId("Boolean"); + public static StringId byteId = new StringId("Byte"); + public static StringId charId = new StringId("Char"); + public static StringId doubleId = new StringId("Double"); + public static StringId shortId = new StringId("Int16"); + public static StringId intId = new StringId("Int32"); + public static StringId longId = new StringId("Int64"); + public static StringId intptrId = new StringId("IntPtr"); + public static StringId objectId = new StringId("Object"); + public static StringId sbyteId = new StringId("SByte"); + public static StringId floatId = new StringId("Single"); + public static StringId stringId = new StringId("String"); + public static StringId ushortId = new StringId("UInt16"); + public static StringId uintId = new StringId("UInt32"); + public static StringId ulongId = new StringId("UInt64"); + public static StringId uintptrId = new StringId("UIntPtr"); + public static StringId voidId = new StringId("Void"); + public static StringId typedReferenceId = new StringId("TypedReference"); + + public static StringId Id(this PrimitiveTypeCode typeCode) + { + switch (typeCode) + { + case PrimitiveTypeCode.Boolean: return boolId; + case PrimitiveTypeCode.Byte: return byteId; + case PrimitiveTypeCode.Char: return charId; + case PrimitiveTypeCode.Double: return doubleId; + case PrimitiveTypeCode.Int16: return shortId; + case PrimitiveTypeCode.Int32: return intId; + case PrimitiveTypeCode.Int64: return longId; + case PrimitiveTypeCode.IntPtr: return intptrId; + case PrimitiveTypeCode.Object: return objectId; + case PrimitiveTypeCode.SByte: return sbyteId; + case PrimitiveTypeCode.Single: return floatId; + case PrimitiveTypeCode.String: return stringId; + case PrimitiveTypeCode.UInt16: return ushortId; + case PrimitiveTypeCode.UInt32: return uintId; + case PrimitiveTypeCode.UInt64: return ulongId; + case PrimitiveTypeCode.UIntPtr: return uintptrId; + case PrimitiveTypeCode.Void: return voidId; + case PrimitiveTypeCode.TypedReference: return typedReferenceId; + default: throw new InternalError("Unhandled type code {0}", typeCode); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/PDB/MetadataPdbReader.cs b/csharp/extractor/Semmle.Extraction.CIL/PDB/MetadataPdbReader.cs new file mode 100644 index 00000000000..33e8460090d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/PDB/MetadataPdbReader.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; + +namespace Semmle.Extraction.PDB +{ + /// + /// A reader of PDB information using System.Reflection.Metadata. + /// This is cross platform, and the future of PDB. + /// + /// PDB information can be in a separate PDB file, or embedded in the DLL. + /// + class MetadataPdbReader : IPdb + { + class SourceFile : ISourceFile + { + public SourceFile(MetadataReader reader, DocumentHandle handle) + { + var doc = reader.GetDocument(handle); + Path = reader.GetString(doc.Name); + } + + public string Path { get; private set; } + + public string Contents => File.Exists(Path) ? File.ReadAllText(Path, System.Text.Encoding.Default) : null; + } + + // Turns out to be very important to keep the MetadataReaderProvider live + // or the reader will crash. + readonly MetadataReaderProvider provider; + readonly MetadataReader reader; + + public MetadataPdbReader(MetadataReaderProvider provider) + { + this.provider = provider; + reader = provider.GetMetadataReader(); + } + + public IEnumerable SourceFiles => reader.Documents.Select(handle => new SourceFile(reader, handle)); + + public IMethod GetMethod(MethodDebugInformationHandle handle) + { + var debugInfo = reader.GetMethodDebugInformation(handle); + + var sequencePoints = debugInfo.GetSequencePoints(). + Where(p => !p.Document.IsNil && !p.IsHidden). + Select(p => new SequencePoint(p.Offset, new Location(new SourceFile(reader, p.Document), p.StartLine, p.StartColumn, p.EndLine, p.EndColumn))). + Where(p => p.Location.File.Path != null). + ToArray(); + + return sequencePoints.Any() ? new Method() { SequencePoints = sequencePoints } : null; + } + + public static MetadataPdbReader CreateFromAssembly(string assemblyPath, PEReader peReader) + { + foreach (var provider in peReader. + ReadDebugDirectory(). + Where(d => d.Type == DebugDirectoryEntryType.EmbeddedPortablePdb). + Select(dirEntry => peReader.ReadEmbeddedPortablePdbDebugDirectoryData(dirEntry))) + { + return new MetadataPdbReader(provider); + } + + try + { + MetadataReaderProvider provider; + string pdbPath; + if (peReader.TryOpenAssociatedPortablePdb(assemblyPath, s => new FileStream(s, FileMode.Open, FileAccess.Read, FileShare.Read), out provider, out pdbPath)) + { + return new MetadataPdbReader(provider); + } + } + + catch (BadImageFormatException) + { + // Something is wrong with the file. + } + catch (FileNotFoundException) + { + // The PDB file was not found. + } + return null; + } + + public void Dispose() + { + provider.Dispose(); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/PDB/NativePdbReader.cs b/csharp/extractor/Semmle.Extraction.CIL/PDB/NativePdbReader.cs new file mode 100644 index 00000000000..0f25b281aab --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/PDB/NativePdbReader.cs @@ -0,0 +1,143 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection.PortableExecutable; +using Microsoft.DiaSymReader; +using System.Reflection.Metadata.Ecma335; +using System.Reflection.Metadata; +using System.IO; +using System.Reflection; + +namespace Semmle.Extraction.PDB +{ + /// + /// A PDB reader using Microsoft.DiaSymReader.Native. + /// This is an unmanaged Windows DLL, which therefore only works on Windows. + /// + class NativePdbReader : IPdb + { + sealed class Document : ISourceFile + { + readonly ISymUnmanagedDocument document; + + public Document(ISymUnmanagedDocument doc) + { + document = doc; + contents = new Lazy(() => + { + bool isEmbedded; + if (document.HasEmbeddedSource(out isEmbedded) == 0 && isEmbedded) + { + var rawContents = document.GetEmbeddedSource().ToArray(); + return System.Text.Encoding.Default.GetString(rawContents); + } + else + { + return File.Exists(Path) ? File.ReadAllText(Path) : null; + } + }); + } + + public override bool Equals(object obj) + { + var otherDoc = obj as Document; + return otherDoc != null && Path.Equals(otherDoc.Path); + } + + public override int GetHashCode() => Path.GetHashCode(); + + public string Path => document.GetName(); + + public override string ToString() => Path; + + readonly Lazy contents; + + public string Contents => contents.Value; + } + + public IEnumerable SourceFiles => reader.GetDocuments().Select(d => new Document(d)); + + public IMethod GetMethod(MethodDebugInformationHandle h) + { + int methodToken = MetadataTokens.GetToken(h.ToDefinitionHandle()); + var method = reader.GetMethod(methodToken); + if (method != null) + { + int count; + if (method.GetSequencePointCount(out count) != 0 || count == 0) + return null; + + var s = method.GetSequencePoints(). + Where(sp => !sp.IsHidden). + Select(sp => new SequencePoint(sp.Offset, new Location(new Document(sp.Document), sp.StartLine, sp.StartColumn, sp.EndLine, sp.EndColumn))). + ToArray(); + + return s.Any() ? new Method { SequencePoints = s } : null; + } + return null; + } + + NativePdbReader(string path) + { + pdbStream = new FileStream(path, FileMode.Open); + var metadataProvider = new MdProvider(); + reader = SymUnmanagedReaderFactory.CreateReader(pdbStream, metadataProvider); + } + + readonly ISymUnmanagedReader5 reader; + readonly FileStream pdbStream; + + public static NativePdbReader CreateFromAssembly(string assemblyPath, PEReader peReader) + { + // The Native PDB reader uses an unmanaged Windows DLL + // so only works on Windows. + if (!Semmle.Util.Win32.IsWindows()) + return null; + + var debugDirectory = peReader.ReadDebugDirectory(); + + foreach (var path in debugDirectory. + Where(d => d.Type == DebugDirectoryEntryType.CodeView). + Select(peReader.ReadCodeViewDebugDirectoryData). + Select(cv => cv.Path). + Where(path => File.Exists(path))) + { + return new NativePdbReader(path); + } + + return null; + } + + public void Dispose() + { + pdbStream.Dispose(); + } + } + + /// + /// This is not used but is seemingly needed in order to use DiaSymReader. + /// + class MdProvider : ISymReaderMetadataProvider + { + public MdProvider() + { + } + + public object GetMetadataImport() => null; + + public unsafe bool TryGetStandaloneSignature(int standaloneSignatureToken, out byte* signature, out int length) => + throw new NotImplementedException(); + + public bool TryGetTypeDefinitionInfo(int typeDefinitionToken, out string namespaceName, out string typeName, out TypeAttributes attributes, out int baseTypeToken) => + throw new NotImplementedException(); + + public bool TryGetTypeDefinitionInfo(int typeDefinitionToken, out string namespaceName, out string typeName, out TypeAttributes attributes) => + throw new NotImplementedException(); + + public bool TryGetTypeReferenceInfo(int typeReferenceToken, out string namespaceName, out string typeName, out int resolutionScopeToken) => + throw new NotImplementedException(); + + public bool TryGetTypeReferenceInfo(int typeReferenceToken, out string namespaceName, out string typeName) => + throw new NotImplementedException(); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/PDB/PdbReader.cs b/csharp/extractor/Semmle.Extraction.CIL/PDB/PdbReader.cs new file mode 100644 index 00000000000..c2f4f94f59f --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/PDB/PdbReader.cs @@ -0,0 +1,151 @@ +using Microsoft.DiaSymReader; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; + +namespace Semmle.Extraction.PDB +{ + /// + /// A sequencepoint is a marker in the source code where you can put a breakpoint, and + /// maps instructions to source code. + /// + public struct SequencePoint + { + /// + /// The byte-offset of the instruction. + /// + public readonly int Offset; + + /// + /// The source location of the instruction. + /// + public readonly Location Location; + + public override string ToString() + { + return string.Format("{0} = {1}", Offset, Location); + } + + public SequencePoint(int offset, Location location) + { + Offset = offset; + Location = location; + } + } + + /// + /// A location in source code. + /// + public sealed class Location + { + /// + /// The file containing the code. + /// + public readonly ISourceFile File; + + /// + /// The span of text within the text file. + /// + public readonly int StartLine, StartColumn, EndLine, EndColumn; + + public override string ToString() + { + return string.Format("({0},{1})-({2},{3})", StartLine, StartColumn, EndLine, EndColumn); + } + + public override bool Equals(object obj) + { + var otherLocation = obj as Location; + + return otherLocation != null && + File.Equals(otherLocation.File) && + StartLine == otherLocation.StartLine && + StartColumn == otherLocation.StartColumn && + EndLine == otherLocation.EndLine && + EndColumn == otherLocation.EndColumn; + } + + public override int GetHashCode() + { + var h1 = StartLine + 37 * (StartColumn + 51 * (EndLine + 97 * EndColumn)); + return File.GetHashCode() + 17 * h1; + } + + public Location(ISourceFile file, int startLine, int startCol, int endLine, int endCol) + { + File = file; + StartLine = startLine; + StartColumn = startCol; + EndLine = endLine; + EndColumn = endCol; + } + } + + public interface IMethod + { + IEnumerable SequencePoints { get; } + Location Location { get; } + } + + class Method : IMethod + { + public IEnumerable SequencePoints { get; set; } + + public Location Location => SequencePoints.First().Location; + } + + /// + /// A source file reference in a PDB file. + /// + public interface ISourceFile + { + string Path { get; } + + /// + /// The contents of the file. + /// This property is needed in case the contents + /// of the file are embedded in the PDB instead of being on the filesystem. + /// + /// null if the contents are unavailable. + /// E.g. if the PDB file exists but the corresponding source files are missing. + /// + string Contents { get; } + } + + /// + /// Wrapper for reading PDB files. + /// This is needed because there are different libraries for dealing with + /// different types of PDB file, even though they share the same file extension. + /// + public interface IPdb : IDisposable + { + /// + /// Gets all source files in this PDB. + /// + IEnumerable SourceFiles { get; } + + /// + /// Look up a method from a given handle. + /// + /// The handle to query. + /// The method information, or null if the method does not have debug information. + IMethod GetMethod(MethodDebugInformationHandle methodHandle); + } + + class PdbReader + { + /// + /// Returns the PDB information associated with an assembly. + /// + /// The path to the assembly. + /// The PE reader for the assembky. + /// A PdbReader, or null if no PDB information is available. + public static IPdb Create(string assemblyPath, PEReader peReader) + { + return (IPdb)MetadataPdbReader.CreateFromAssembly(assemblyPath, peReader) ?? + NativePdbReader.CreateFromAssembly(assemblyPath, peReader); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Properties/AssemblyInfo.cs b/csharp/extractor/Semmle.Extraction.CIL/Properties/AssemblyInfo.cs new file mode 100644 index 00000000000..110feb8de0d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Properties/AssemblyInfo.cs @@ -0,0 +1,35 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Semmle.Extraction.CIL")] +[assembly: AssemblyDescription("Semme CIL extractor.")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Semmle.Extraction.CIL")] +[assembly: AssemblyCopyright("Copyright © Semmle 2017")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("a23d9ec2-8aae-43da-97cb-579f640b89cd")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/csharp/extractor/Semmle.Extraction.CIL/Semmle.Extraction.CIL.csproj b/csharp/extractor/Semmle.Extraction.CIL/Semmle.Extraction.CIL.csproj new file mode 100644 index 00000000000..80808686b4b --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Semmle.Extraction.CIL.csproj @@ -0,0 +1,26 @@ + + + + netcoreapp2.0 + Semmle.Extraction.CIL + Semmle.Extraction.CIL + false + true + + + + + + + + + + + + + + + + + + diff --git a/csharp/extractor/Semmle.Extraction.CIL/Tuples.cs b/csharp/extractor/Semmle.Extraction.CIL/Tuples.cs new file mode 100644 index 00000000000..f75adc2963c --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Tuples.cs @@ -0,0 +1,209 @@ +using Semmle.Extraction.CIL.Entities; + +namespace Semmle.Extraction.CIL +{ + + internal static class Tuples + { + internal static Tuple assemblies(Assembly assembly, File file, string identifier, string name, string version) => + new Tuple("assemblies", assembly, file, identifier, name, version); + + internal static Tuple cil_abstract(IMember method) => + new Tuple("cil_abstract", method); + + internal static Tuple cil_adder(IEvent member, IMethod method) => + new Tuple("cil_adder", member, method); + + internal static Tuple cil_access(IInstruction i, IEntity m) => + new Tuple("cil_access", i, m); + + internal static Tuple cil_attribute(IAttribute attribute, IEntity @object, IMethod constructor) => + new Tuple("cil_attribute", attribute, @object, constructor); + + internal static Tuple cil_attribute_named_argument(IAttribute attribute, string name, string value) => + new Tuple("cil_attribute_named_argument", attribute, name, value); + + internal static Tuple cil_attribute_positional_argument(IAttribute attribute, int index, string value) => + new Tuple("cil_attribute_positional_argument", attribute, index, value); + + internal static Tuple cil_array_type(IArrayType array, IType element, int rank) => + new Tuple("cil_array_type", array, element, rank); + + internal static Tuple cil_base_class(IType t, IType @base) => + new Tuple("cil_base_class", t, @base); + + internal static Tuple cil_base_interface(IType t, IType @base) => + new Tuple("cil_base_interface", t, @base); + + internal static Tuple cil_class(IMember method) => + new Tuple("cil_class", method); + + internal static Tuple cil_event(IEvent e, IType parent, string name, IType type) => + new Tuple("cil_event", e, parent, name, type); + + internal static Tuple cil_field(IField field, IType parent, string name, IType fieldType) => + new Tuple("cil_field", field, parent, name, fieldType); + + internal static Tuple cil_getter(IProperty member, IMethod method) => + new Tuple("cil_getter", member, method); + + internal static Tuple cil_handler(IExceptionRegion region, IMethodImplementation method, int index, int kind, + IInstruction region_start, + IInstruction region_end, + IInstruction handler_start) => + new Tuple("cil_handler", region, method, index, kind, region_start, region_end, handler_start); + + internal static Tuple cil_handler_filter(IExceptionRegion region, IInstruction filter_start) => + new Tuple("cil_handler_filter", region, filter_start); + + internal static Tuple cil_handler_type(IExceptionRegion region, Type t) => + new Tuple("cil_handler_type", region, t); + + internal static Tuple cil_implements(IMethod derived, IMethod declaration) => + new Tuple("cil_implements", derived, declaration); + + internal static Tuple cil_instruction(IInstruction instruction, int opcode, int index, IMethodImplementation parent) => + new Tuple("cil_instruction", instruction, opcode, index, parent); + + internal static Tuple cil_instruction_location(Instruction i, ILocation loc) => + new Tuple("cil_instruction_location", i, loc); + + internal static Tuple cil_interface(IMember method) => + new Tuple("cil_interface", method); + + internal static Tuple cil_internal(IMember modifiable) => + new Tuple("cil_internal", modifiable); + + internal static Tuple cil_jump(IInstruction from, IInstruction to) => + new Tuple("cil_jump", from, to); + + internal static Tuple cil_local_variable(ILocal l, IMethodImplementation m, int i, Type t) => + new Tuple("cil_local_variable", l, m, i, t); + + internal static Tuple cil_method(IMethod method, string name, IType declType, IType returnType) => + new Tuple("cil_method", method, name, declType, returnType); + + internal static Tuple cil_method_implementation(IMethodImplementation impl, IMethod method, IAssembly assembly) => + new Tuple("cil_method_implementation", impl, method, assembly); + + internal static Tuple cil_method_location(IMethod m, ILocation a) => + new Tuple("cil_method_location", m, a); + + internal static Tuple cil_method_source_declaration(IMethod method, IMethod sourceDecl) => + new Tuple("cil_method_source_declaration", method, sourceDecl); + + internal static Tuple cil_method_stack_size(IMethodImplementation method, int stackSize) => + new Tuple("cil_method_stack_size", method, stackSize); + + internal static Tuple cil_newslot(IMethod method) => + new Tuple("cil_newslot", method); + + internal static Tuple cil_parameter(IParameter p, IMethod m, int i, IType t) => + new Tuple("cil_parameter", p, m, i, t); + + internal static Tuple cil_parameter_in(IParameter p) => + new Tuple("cil_parameter_in", p); + + internal static Tuple cil_parameter_out(IParameter p) => + new Tuple("cil_parameter_out", p); + + internal static Tuple cil_pointer_type(IPointerType t, IType pointee) => + new Tuple("cil_pointer_type", t, pointee); + + internal static Tuple cil_private(IMember modifiable) => + new Tuple("cil_private", modifiable); + + internal static Tuple cil_protected(IMember modifiable) => + new Tuple("cil_protected", modifiable); + + internal static Tuple cil_property(IProperty p, IType parent, string name, IType propType) => + new Tuple("cil_property", p, parent, name, propType); + + internal static Tuple cil_public(IMember modifiable) => + new Tuple("cil_public", modifiable); + + internal static Tuple cil_raiser(IEvent member, IMethod method) => + new Tuple("cil_raiser", member, method); + + internal static Tuple cil_requiresecobject(IMethod method) => + new Tuple("cil_requiresecobject", method); + + internal static Tuple cil_remover(IEvent member, IMethod method) => + new Tuple("cil_remover", member, method); + + internal static Tuple cil_sealed(IMember modifiable) => + new Tuple("cil_sealed", modifiable); + + internal static Tuple cil_security(IMember method) => + new Tuple("cil_security", method); + + internal static Tuple cil_setter(IProperty member, IMethod method) => + new Tuple("cil_setter", member, method); + + internal static Tuple cil_specialname(IMethod method) => + new Tuple("cil_specialname", method); + + internal static Tuple cil_static(IMember modifiable) => + new Tuple("cil_static", modifiable); + + internal static Tuple cil_switch(IInstruction from, int index, IInstruction to) => + new Tuple("cil_switch", from, index, to); + + internal static Tuple cil_type(IType t, string name, CilTypeKind kind, ITypeContainer parent, IType sourceDecl) => + new Tuple("cil_type", t, name, (int)kind, parent, sourceDecl); + + internal static Tuple cil_type_argument(ITypeContainer constructedTypeOrMethod, int index, IType argument) => + new Tuple("cil_type_argument", constructedTypeOrMethod, index, argument); + + internal static Tuple cil_type_location(IType t, IAssembly a) => + new Tuple("cil_type_location", t, a); + + internal static Tuple cil_type_parameter(ITypeContainer unboundTypeOrMethod, int index, ITypeParameter parameter) => + new Tuple("cil_type_parameter", unboundTypeOrMethod, index, parameter); + + internal static Tuple cil_typeparam_covariant(ITypeParameter p) => + new Tuple("cil_typeparam_covariant", p); + + internal static Tuple cil_typeparam_contravariant(ITypeParameter p) => + new Tuple("cil_typeparam_contravariant", p); + + internal static Tuple cil_typeparam_class(ITypeParameter p) => + new Tuple("cil_typeparam_class", p); + + internal static Tuple cil_typeparam_constraint(ITypeParameter p, IType constraint) => + new Tuple("cil_typeparam_constraint", p, constraint); + + internal static Tuple cil_typeparam_new(ITypeParameter p) => + new Tuple("cil_typeparam_new", p); + + internal static Tuple cil_typeparam_struct(ITypeParameter p) => + new Tuple("cil_typeparam_struct", p); + + internal static Tuple cil_value(IInstruction i, string value) => + new Tuple("cil_value", i, value); + + internal static Tuple cil_virtual(IMethod method) => + new Tuple("cil_virtual", method); + + internal static Tuple containerparent(IFolder parent, IFileOrFolder child) => + new Tuple("containerparent", parent, child); + + internal static Tuple files(IFile file, string fullName, string name, string extension) => + new Tuple("files", file, fullName, name, extension, 0); + + internal static Tuple file_extraction_mode(IFile file, int mode) => + new Tuple("file_extraction_mode", file, mode); + + internal static Tuple folders(IFolder folder, string path, string name) => + new Tuple("folders", folder, path, name); + + internal static Tuple locations_default(ISourceLocation label, IFile file, int startLine, int startCol, int endLine, int endCol) => + new Tuple("locations_default", label, file, startLine, startCol, endLine, endCol); + + internal static Tuple namespaces(INamespace ns, string name) => + new Tuple("namespaces", ns, name); + + internal static Tuple parent_namespace(ITypeContainer child, INamespace parent) => + new Tuple("parent_namespace", child, parent); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Driver/Driver.cs b/csharp/extractor/Semmle.Extraction.CSharp.Driver/Driver.cs new file mode 100644 index 00000000000..e2892678041 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Driver/Driver.cs @@ -0,0 +1,13 @@ +namespace Semmle.Extraction.CSharp +{ + /// + /// A command-line driver for the extractor. + /// + public class Driver + { + static int Main(string[] args) + { + return (int)Extractor.Run(args); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/AssemblyInfo.cs b/csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/AssemblyInfo.cs new file mode 100644 index 00000000000..4ef49f38f26 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/AssemblyInfo.cs @@ -0,0 +1,35 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Semmle.Extraction.CSharp.Driver")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Semmle.Extraction.CSharp.Driver")] +[assembly: AssemblyCopyright("Copyright © 2015")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("c02a2b0e-8884-4b82-8275-ea282403a775")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Driver/Semmle.Extraction.CSharp.Driver.csproj b/csharp/extractor/Semmle.Extraction.CSharp.Driver/Semmle.Extraction.CSharp.Driver.csproj new file mode 100644 index 00000000000..dcaccd3179d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Driver/Semmle.Extraction.CSharp.Driver.csproj @@ -0,0 +1,15 @@ + + + + Exe + netcoreapp2.0 + Semmle.Extraction.CSharp.Driver + Semmle.Extraction.CSharp.Driver + false + + + + + + + diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs new file mode 100644 index 00000000000..db2664bf4c9 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs @@ -0,0 +1,184 @@ +using System.Collections.Generic; +using System.Linq; +using System.IO; +using System; + +namespace Semmle.BuildAnalyser +{ + /// + /// Manages the set of assemblies. + /// Searches for assembly DLLs, indexes them and provides + /// a lookup facility from assembly ID to filename. + /// + class AssemblyCache + { + /// + /// Locate all reference files and index them. + /// + /// Directories to search. + /// Callback for progress. + public AssemblyCache(IEnumerable dirs, IProgressMonitor progress) + { + foreach (var dir in dirs) + { + progress.FindingFiles(dir); + AddReferenceDirectory(dir); + } + IndexReferences(); + } + + /// + /// Finds all assemblies nested within a directory + /// and adds them to its index. + /// (Indexing is performed at a later stage by IndexReferences()). + /// + /// The directory to index. + /// The number of DLLs within this directory. + int AddReferenceDirectory(string dir) + { + int count = 0; + foreach (var dll in new DirectoryInfo(dir).EnumerateFiles("*.dll", SearchOption.AllDirectories)) + { + dlls.Add(dll.FullName); + ++count; + } + return count; + } + + /// + /// Indexes all DLLs we have located. + /// Because this is a potentially time-consuming operation, it is put into a separate stage. + /// + void IndexReferences() + { + // Read all of the files + foreach (var filename in dlls) + { + var info = AssemblyInfo.ReadFromFile(filename); + + if (info.Valid) + { + assemblyInfo[filename] = info; + } + else + { + failedDlls.Add(filename); + } + } + + // Index "assemblyInfo" by version string + // The OrderBy is used to ensure that we by default select the highest version number. + foreach (var info in assemblyInfo.Values.OrderBy(info => info.Id)) + { + foreach (var index in info.IndexStrings) + references[index] = info; + } + } + + /// + /// The number of DLLs which are assemblies. + /// + public int AssemblyCount => assemblyInfo.Count; + + /// + /// The number of DLLs which weren't assemblies. (E.g. C++). + /// + public int NonAssemblyCount => failedDlls.Count; + + /// + /// Given an assembly id, determine its full info. + /// + /// The given assembly id. + /// The information about the assembly. + public AssemblyInfo ResolveReference(string id) + { + // Fast path if we've already seen this before. + if (failedReferences.Contains(id)) + return AssemblyInfo.Invalid; + + var query = AssemblyInfo.MakeFromId(id); + id = query.Id; // Sanitise the id. + + // Look up the id in our references map. + AssemblyInfo result; + if (references.TryGetValue(id, out result)) + { + // The string is in the references map. + return result; + } + else + { + // Attempt to load the reference from the GAC. + try + { + var loadedAssembly = System.Reflection.Assembly.ReflectionOnlyLoad(id); + + if (loadedAssembly != null) + { + // The assembly was somewhere we haven't indexed before. + // Add this assembly to our index so that subsequent lookups are faster. + + result = AssemblyInfo.MakeFromAssembly(loadedAssembly); + references[id] = result; + assemblyInfo[loadedAssembly.Location] = result; + return result; + } + } + catch (FileNotFoundException) + { + // A suitable assembly could not be found + } + catch (FileLoadException) + { + // The assembly cannot be loaded for some reason + // e.g. The name is malformed. + } + catch (PlatformNotSupportedException) + { + // .NET Core does not have a GAC. + } + + // Fallback position - locate the assembly by its lower-case name only. + var asmName = query.Name.ToLowerInvariant(); + + if (references.TryGetValue(asmName, out result)) + { + references[asmName] = result; // Speed up the next time the same string is resolved + return result; + } + + failedReferences.Add(id); // Fail early next time + + return AssemblyInfo.Invalid; + } + } + + /// + /// All the assemblies we have indexed. + /// + public IEnumerable AllAssemblies => assemblyInfo.Select(a => a.Value); + + /// + /// Retrieve the assembly info of a pre-cached assembly. + /// + /// The filename to query. + /// The assembly info. + public AssemblyInfo GetAssemblyInfo(string filepath) => assemblyInfo[filepath]; + + // List of pending DLLs to index. + readonly List dlls = new List(); + + // Map from filename to assembly info. + readonly Dictionary assemblyInfo = new Dictionary(); + + // List of DLLs which are not assemblies. + // We probably don't need to keep this + readonly List failedDlls = new List(); + + // Map from assembly id (in various formats) to the full info. + readonly Dictionary references = new Dictionary(); + + // Set of failed assembly ids. + readonly HashSet failedReferences = new HashSet(); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyInfo.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyInfo.cs new file mode 100644 index 00000000000..cacc2bca945 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyInfo.cs @@ -0,0 +1,179 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Security.Cryptography; +using System.Text; + +namespace Semmle.BuildAnalyser +{ + /// + /// Stores information about an assembly file (DLL). + /// + sealed class AssemblyInfo + { + /// + /// The file containing the assembly. + /// + public string Filename { get; private set; } + + /// + /// Was the information correctly determined? + /// + public bool Valid { get; private set; } + + /// + /// The short name of this assembly. + /// + public string Name { get; private set; } + + /// + /// The version number of this assembly. + /// + public System.Version Version { get; private set; } + + /// + /// The public key token of the assembly. + /// + public string PublicKeyToken { get; private set; } + + /// + /// The culture. + /// + public string Culture { get; private set; } + + /// + /// Get/parse a canonical ID of this assembly. + /// + public string Id + { + get + { + var result = Name; + if (Version != null) + result = string.Format("{0}, Version={1}", result, Version); + if (Culture != null) + result = string.Format("{0}, Culture={1}", result, Culture); + if (PublicKeyToken != null) + result = string.Format("{0}, PublicKeyToken={1}", result, PublicKeyToken); + return result; + } + + private set + { + var sections = value.Split(new string[] { ", " }, StringSplitOptions.None); + + Name = sections.First(); + + foreach (var section in sections.Skip(1)) + { + if (section.StartsWith("Version=")) + Version = new Version(section.Substring(8)); + else if (section.StartsWith("Culture=")) + Culture = section.Substring(8); + else if (section.StartsWith("PublicKeyToken=")) + PublicKeyToken = section.Substring(15); + // else: Some other field like processorArchitecture - ignore. + } + + } + } + + public override string ToString() => Id; + + /// + /// Gets a list of canonical search strings for this assembly. + /// + public IEnumerable IndexStrings + { + get + { + yield return Id; + if (Version != null) + { + if (Culture != null) yield return string.Format("{0}, Version={1}, Culture={2}", Name, Version, Culture); + yield return string.Format("{0}, Version={1}", Name, Version); + } + yield return Name; + yield return Name.ToLowerInvariant(); + } + } + + /// + /// Get an invalid assembly info (Valid==false). + /// + public static AssemblyInfo Invalid { get; } = new AssemblyInfo(); + + private AssemblyInfo() { } + + /// + /// Get AssemblyInfo from a loaded Assembly. + /// + /// The assembly. + /// Info about the assembly. + public static AssemblyInfo MakeFromAssembly(Assembly assembly) => new AssemblyInfo() { Valid = true, Filename = assembly.Location, Id = assembly.FullName }; + + /// + /// Parse an assembly name/Id into an AssemblyInfo, + /// populating the available fields and leaving the others null. + /// + /// The assembly name/Id. + /// The deconstructed assembly info. + public static AssemblyInfo MakeFromId(string id) => new AssemblyInfo() { Valid = true, Id = id }; + + /// + /// Reads the assembly info from a file. + /// This uses System.Reflection.Metadata, which is a very performant and low-level + /// library. This is very convenient when scanning hundreds of DLLs at a time. + /// + /// The full filename of the assembly. + /// The information about the assembly. + public static AssemblyInfo ReadFromFile(string filename) + { + var result = new AssemblyInfo() { Filename = filename }; + try + { + /* This method is significantly faster and more lightweight than using + * System.Reflection.Assembly.ReflectionOnlyLoadFrom. It also allows + * loading the same assembly from different locations. + */ + using (var pereader = new System.Reflection.PortableExecutable.PEReader(new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))) + { + var metadata = pereader.GetMetadata(); + unsafe + { + var reader = new System.Reflection.Metadata.MetadataReader(metadata.Pointer, metadata.Length); + var def = reader.GetAssemblyDefinition(); + + // This is how you compute the public key token from the full public key. + // The last 8 bytes of the SHA1 of the public key. + var publicKey = reader.GetBlobBytes(def.PublicKey); + var publicKeyToken = sha1.ComputeHash(publicKey); + var publicKeyString = new StringBuilder(); + foreach (var b in publicKeyToken.Skip(12).Reverse()) + publicKeyString.AppendFormat("{0:x2}", b); + + result.Name = reader.GetString(def.Name); + result.Version = def.Version; + result.Culture = def.Culture.IsNil ? "neutral" : reader.GetString(def.Culture); + result.PublicKeyToken = publicKeyString.ToString(); + result.Valid = true; + } + } + } + catch (BadImageFormatException) + { + // The DLL wasn't an assembly -> result.Valid = false. + } + catch (InvalidOperationException) + { + // Some other failure -> result.Valid = false. + } + + return result; + } + + static readonly SHA1CryptoServiceProvider sha1 = new SHA1CryptoServiceProvider(); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs new file mode 100644 index 00000000000..864316ac60b --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs @@ -0,0 +1,312 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using Semmle.Util; +using Semmle.Extraction.CSharp.Standalone; + +namespace Semmle.BuildAnalyser +{ + /// + /// The output of a build analysis. + /// + interface IBuildAnalysis + { + /// + /// Full filepaths of external references. + /// + IEnumerable ReferenceFiles { get; } + + /// + /// Full filepaths of C# source files from project files. + /// + IEnumerable ProjectSourceFiles { get; } + + /// + /// Full filepaths of C# source files in the filesystem. + /// + IEnumerable AllSourceFiles { get; } + + /// + /// The assembly IDs which could not be resolved. + /// + IEnumerable UnresolvedReferences { get; } + + /// + /// List of source files referenced by projects but + /// which were not found in the filesystem. + /// + IEnumerable MissingSourceFiles { get; } + } + + /// + /// Main implementation of the build analysis. + /// + class BuildAnalysis : IBuildAnalysis + { + readonly AssemblyCache assemblyCache; + readonly NugetPackages nuget; + readonly IProgressMonitor progressMonitor; + HashSet usedReferences = new HashSet(); + readonly HashSet usedSources = new HashSet(); + readonly HashSet missingSources = new HashSet(); + readonly Dictionary unresolvedReferences = new Dictionary(); + readonly DirectoryInfo sourceDir; + int failedProjects, succeededProjects; + readonly string[] allSources; + int conflictedReferences = 0; + + /// + /// Performs a C# build analysis. + /// + /// Analysis options from the command line. + /// Display of analysis progress. + public BuildAnalysis(Options options, IProgressMonitor progress) + { + progressMonitor = progress; + sourceDir = new DirectoryInfo(options.SrcDir); + + progressMonitor.FindingFiles(options.SrcDir); + + allSources = sourceDir.GetFiles("*.cs", SearchOption.AllDirectories). + Select(d => d.FullName). + Where(d => !options.ExcludesFile(d)). + ToArray(); + + var dllDirNames = options.DllDirs.Select(Path.GetFullPath); + + if (options.UseNuGet) + { + nuget = new NugetPackages(sourceDir.FullName); + ReadNugetFiles(); + dllDirNames = dllDirNames.Concat(Enumerators.Singleton(nuget.PackageDirectory)); + } + + // Find DLLs in the .Net Framework + if (options.ScanNetFrameworkDlls) + { + dllDirNames = dllDirNames.Concat(Runtime.Runtimes.Take(1)); + } + + assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress); + + // Analyse all .csproj files in the source tree. + if (options.SolutionFile != null) + { + AnalyseSolution(options.SolutionFile); + } + else if (options.AnalyseCsProjFiles) + { + AnalyseProjectFiles(); + } + + if (!options.AnalyseCsProjFiles) + { + usedReferences = new HashSet(assemblyCache.AllAssemblies.Select(a => a.Filename)); + } + + ResolveConflicts(); + + if (options.UseMscorlib) + { + UseReference(typeof(object).Assembly.Location); + } + + // Output the findings + foreach (var r in usedReferences) + { + progressMonitor.ResolvedReference(r); + } + + foreach (var r in unresolvedReferences) + { + progressMonitor.UnresolvedReference(r.Key, r.Value); + } + + progressMonitor.Summary( + AllSourceFiles.Count(), + ProjectSourceFiles.Count(), + MissingSourceFiles.Count(), + ReferenceFiles.Count(), + UnresolvedReferences.Count(), + conflictedReferences, + succeededProjects + failedProjects, + failedProjects); + } + + /// + /// Resolves conflicts between all of the resolved references. + /// If the same assembly name is duplicated with different versions, + /// resolve to the higher version number. + /// + void ResolveConflicts() + { + var sortedReferences = usedReferences. + Select(r => assemblyCache.GetAssemblyInfo(r)). + OrderBy(r => r.Version). + ToArray(); + + Dictionary finalAssemblyList = new Dictionary(); + + // Pick the highest version for each assembly name + foreach (var r in sortedReferences) + finalAssemblyList[r.Name] = r; + + // Update the used references list + usedReferences = new HashSet(finalAssemblyList.Select(r => r.Value.Filename)); + + // Report the results + foreach (var r in sortedReferences) + { + var resolvedInfo = finalAssemblyList[r.Name]; + if (resolvedInfo.Version != r.Version) + { + progressMonitor.ResolvedConflict(r.Id, resolvedInfo.Id); + ++conflictedReferences; + } + } + } + + /// + /// Find and restore NuGet packages. + /// + void ReadNugetFiles() + { + nuget.FindPackages(); + nuget.InstallPackages(progressMonitor); + } + + /// + /// Store that a particular reference file is used. + /// + /// The filename of the reference. + void UseReference(string reference) + { + usedReferences.Add(reference); + } + + /// + /// Store that a particular source file is used (by a project file). + /// + /// The source file. + void UseSource(FileInfo sourceFile) + { + if (sourceFile.Exists) + { + usedSources.Add(sourceFile.FullName); + } + else + { + missingSources.Add(sourceFile.FullName); + } + } + + /// + /// The list of resolved reference files. + /// + public IEnumerable ReferenceFiles => this.usedReferences; + + /// + /// The list of source files used in projects. + /// + public IEnumerable ProjectSourceFiles => usedSources; + + /// + /// All of the source files in the source directory. + /// + public IEnumerable AllSourceFiles => allSources; + + /// + /// List of assembly IDs which couldn't be resolved. + /// + public IEnumerable UnresolvedReferences => this.unresolvedReferences.Select(r => r.Key); + + /// + /// List of source files which were mentioned in project files but + /// do not exist on the file system. + /// + public IEnumerable MissingSourceFiles => missingSources; + + /// + /// Record that a particular reference couldn't be resolved. + /// Note that this records at most one project file per missing reference. + /// + /// The assembly ID. + /// The project file making the reference. + void UnresolvedReference(string id, string projectFile) + { + unresolvedReferences[id] = projectFile; + } + + /// + /// Performs an analysis of all .csproj files. + /// + void AnalyseProjectFiles() + { + AnalyseProjectFiles(sourceDir.GetFiles("*.csproj", SearchOption.AllDirectories)); + } + + /// + /// Reads all the source files and references from the given list of projects. + /// + /// The list of projects to analyse. + void AnalyseProjectFiles(FileInfo[] projectFiles) + { + progressMonitor.AnalysingProjectFiles(projectFiles.Count()); + + foreach (var proj in projectFiles) + { + try + { + var csProj = new CsProjFile(proj); + + foreach (var @ref in csProj.References) + { + AssemblyInfo resolved = assemblyCache.ResolveReference(@ref); + if (!resolved.Valid) + { + UnresolvedReference(@ref, proj.FullName); + } + else + { + UseReference(resolved.Filename); + } + } + + foreach (var src in csProj.Sources) + { + // Make a note of which source files the projects use. + // This information doesn't affect the build but is dumped + // as diagnostic output. + UseSource(new FileInfo(src)); + } + ++succeededProjects; + } + catch (Exception ex) + { + ++failedProjects; + progressMonitor.FailedProjectFile(proj.FullName, ex.Message); + } + } + } + + /// + /// Delete packages directory. + /// + public void Cleanup() + { + if (nuget != null) nuget.Cleanup(progressMonitor); + } + + /// + /// Analyse all project files in a given solution only. + /// + /// The filename of the solution. + public void AnalyseSolution(string solutionFile) + { + var sln = new SolutionFile(solutionFile); + AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).ToArray()); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs new file mode 100644 index 00000000000..f866a435180 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs @@ -0,0 +1,120 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml; + +namespace Semmle.BuildAnalyser +{ + /// + /// Represents a .csproj file and reads information from it. + /// + class CsProjFile + { + /// + /// Reads the .csproj file. + /// + /// The .csproj file. + public CsProjFile(FileInfo filename) + { + try + { + // This can fail if the .csproj is invalid or has + // unrecognised content or is the wrong version. + // This currently always fails on Linux because + // Microsoft.Build is not cross platform. + ReadMsBuildProject(filename); + } + catch + { + // There was some reason why the project couldn't be loaded. + // Fall back to reading the Xml document directly. + // This method however doesn't handle variable expansion. + ReadProjectFileAsXml(filename); + } + } + + /// + /// Read the .csproj file using Microsoft Build. + /// This occasionally fails if the project file is incompatible for some reason, + /// and there seems to be no way to make it succeed. Fails on Linux. + /// + /// The file to read. + public void ReadMsBuildProject(FileInfo filename) + { + var msbuildProject = new Microsoft.Build.Execution.ProjectInstance(filename.FullName); + + references = msbuildProject. + Items. + Where(item => item.ItemType == "Reference"). + Select(item => item.EvaluatedInclude). + ToArray(); + + csFiles = msbuildProject.Items + .Where(item => item.ItemType == "Compile") + .Select(item => item.GetMetadataValue("FullPath")) + .Where(fn => fn.EndsWith(".cs")) + .ToArray(); + } + + /// + /// Reads the .csproj file directly as XML. + /// This doesn't handle variables etc, and should only used as a + /// fallback if ReadMsBuildProject() fails. + /// + /// The .csproj file. + public void ReadProjectFileAsXml(FileInfo filename) + { + var projFile = new XmlDocument(); + var mgr = new XmlNamespaceManager(projFile.NameTable); + mgr.AddNamespace("msbuild", "http://schemas.microsoft.com/developer/msbuild/2003"); + projFile.Load(filename.FullName); + var projDir = filename.Directory; + var root = projFile.DocumentElement; + + references = + root.SelectNodes("/msbuild:Project/msbuild:ItemGroup/msbuild:Reference/@Include", mgr). + NodeList(). + Select(node => node.Value). + ToArray(); + + var relativeCsIncludes = + root.SelectNodes("/msbuild:Project/msbuild:ItemGroup/msbuild:Compile/@Include", mgr). + NodeList(). + Select(node => node.Value). + ToArray(); + + csFiles = relativeCsIncludes. + Select(cs => Path.DirectorySeparatorChar == '/' ? cs.Replace("\\", "/") : cs). + Select(f => Path.GetFullPath(Path.Combine(projDir.FullName, f))). + ToArray(); + } + + string[] references; + string[] csFiles; + + /// + /// The list of references as a list of assembly IDs. + /// + public IEnumerable References => references; + + /// + /// The list of C# source files in full path format. + /// + public IEnumerable Sources => csFiles; + } + + static class XmlNodeHelper + { + /// + /// Helper to convert an XmlNodeList into an IEnumerable. + /// This allows it to be used with Linq. + /// + /// The list to convert. + /// A more useful data type. + public static IEnumerable NodeList(this XmlNodeList list) + { + foreach (var i in list) + yield return i as XmlNode; + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs new file mode 100644 index 00000000000..067f0a84cfe --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs @@ -0,0 +1,195 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; + +namespace Semmle.BuildAnalyser +{ + /// + /// Manage the downloading of NuGet packages. + /// Locates packages in a source tree and downloads all of the + /// referenced assemblies to a temp folder. + /// + class NugetPackages + { + /// + /// Create the package manager for a specified source tree. + /// + /// The source directory. + public NugetPackages(string sourceDir) + { + SourceDirectory = sourceDir; + PackageDirectory = computeTempDirectory(sourceDir); + + // Expect nuget.exe to be in a `nuget` directory under the directory containing this exe. + var currentAssembly = System.Reflection.Assembly.GetExecutingAssembly().Location; + nugetExe = Path.Combine(Path.GetDirectoryName(currentAssembly), "nuget", "nuget.exe"); + + if (!File.Exists(nugetExe)) + throw new FileNotFoundException(string.Format("NuGet could not be found at {0}", nugetExe)); + } + + /// + /// Locate all NuGet packages but don't download them yet. + /// + public void FindPackages() + { + packages = new DirectoryInfo(SourceDirectory). + EnumerateFiles("packages.config", SearchOption.AllDirectories). + ToArray(); + } + + // List of package files to download. + FileInfo[] packages; + + /// + /// The list of package files. + /// + public IEnumerable PackageFiles => packages; + + // Whether to delete the packages directory prior to each run. + // Makes each build more reproducible. + const bool cleanupPackages = true; + + public void Cleanup(IProgressMonitor pm) + { + var packagesDirectory = new DirectoryInfo(PackageDirectory); + + if (packagesDirectory.Exists) + { + try + { + packagesDirectory.Delete(true); + } + catch (System.IO.IOException ex) + { + pm.Warning(string.Format("Couldn't delete package directory - it's probably held open by something else: {0}", ex.Message)); + } + } + } + + /// + /// Download the packages to the temp folder. + /// + /// The progress monitor used for reporting errors etc. + public void InstallPackages(IProgressMonitor pm) + { + if (cleanupPackages) + { + Cleanup(pm); + } + + var packagesDirectory = new DirectoryInfo(PackageDirectory); + + if (!Directory.Exists(PackageDirectory)) + { + packagesDirectory.Create(); + } + + foreach (var package in packages) + { + RestoreNugetPackage(package.FullName, pm); + } + } + + /// + /// The source directory used. + /// + public string SourceDirectory + { + get; + private set; + } + + /// + /// The computed packages directory. + /// This will be in the Temp location + /// so as to not trample the source tree. + /// + public string PackageDirectory + { + get; + private set; + } + + readonly SHA1CryptoServiceProvider sha1 = new SHA1CryptoServiceProvider(); + + /// + /// Computes a unique temp directory for the packages associated + /// with this source tree. Use a SHA1 of the directory name. + /// + /// + /// The full path of the temp directory. + string computeTempDirectory(string srcDir) + { + var bytes = Encoding.Unicode.GetBytes(srcDir); + + var sha = sha1.ComputeHash(bytes); + var sb = new StringBuilder(); + foreach (var b in sha.Take(8)) + sb.AppendFormat("{0:x2}", b); + + return Path.Combine(Path.GetTempPath(), "Semmle", "packages", sb.ToString()); + } + + /// + /// Restore all files in a specified package. + /// + /// The package file. + /// Where to log progress/errors. + void RestoreNugetPackage(string package, IProgressMonitor pm) + { + pm.NugetInstall(package); + + /* Use nuget.exe to install a package. + * Note that there is a clutch of NuGet assemblies which could be used to + * invoke this directly, which would arguably be nicer. However they are + * really unwieldy and this solution works for now. + */ + + string exe, args; + if (Util.Win32.IsWindows()) + { + exe = nugetExe; + args = string.Format("install -OutputDirectory {0} {1}", PackageDirectory, package); + } + else + { + exe = "mono"; + args = string.Format("{0} install -OutputDirectory {1} {2}", nugetExe, PackageDirectory, package); + } + + var pi = new ProcessStartInfo(exe, args) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + }; + + try + { + using (var p = Process.Start(pi)) + { + string output = p.StandardOutput.ReadToEnd(); + string error = p.StandardError.ReadToEnd(); + + p.WaitForExit(); + if (p.ExitCode != 0) + { + pm.FailedNugetCommand(pi.FileName, pi.Arguments, output + error); + } + } + } + catch (Exception e) + when (e is System.ComponentModel.Win32Exception || e is FileNotFoundException) + { + pm.FailedNugetCommand(pi.FileName, pi.Arguments, e.Message); + } + } + + readonly string nugetExe; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Options.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Options.cs new file mode 100644 index 00000000000..89cc18007f0 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Options.cs @@ -0,0 +1,178 @@ +using Semmle.Util.Logging; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Semmle.Util; + +namespace Semmle.Extraction.CSharp.Standalone +{ + /// + /// The options controlling standalone extraction. + /// + public sealed class Options : CommonOptions + { + public override bool handleFlag(string key, bool value) + { + switch(key) + { + case "silent": + Verbosity = value ? Verbosity.Off : Verbosity.Info; + return true; + case "help": + Help = true; + return true; + case "dry-run": + SkipExtraction = value; + return true; + case "skip-nuget": + UseNuGet = !value; + return true; + case "all-references": + AnalyseCsProjFiles = !value; + return true; + case "stdlib": + UseMscorlib = value; + return true; + case "skip-dotnet": + ScanNetFrameworkDlls = !value; + return true; + default: + return base.handleFlag(key, value); + } + } + + public override bool handleOption(string key, string value) + { + switch(key) + { + case "exclude": + Excludes.Add(value); + return true; + case "references": + DllDirs.Add(value); + return true; + default: + return base.handleOption(key, value); + } + } + + public override bool handleArgument(string arg) + { + SolutionFile = arg; + var fi = new FileInfo(SolutionFile); + if (!fi.Exists) + { + System.Console.WriteLine("Error: The solution {0} does not exist", fi.FullName); + Errors = true; + } + return true; + } + + public override void invalidArgument(string argument) + { + System.Console.WriteLine($"Error: Invalid argument {argument}"); + Errors = true; + } + + /// + /// Files/patterns to exclude. + /// + public IList Excludes = new List(); + + /// + /// The number of concurrent threads to use. + /// + public int NumberOfThreads = Semmle.Extraction.Extractor.DefaultNumberOfThreads; + + /// + /// The directory containing the source code; + /// + public readonly string SrcDir = System.IO.Directory.GetCurrentDirectory(); + + /// + /// Whether to analyse NuGet packages. + /// + public bool UseNuGet = true; + + /// + /// Directories to search DLLs in. + /// + public IList DllDirs = new List(); + + /// + /// Whether to search the .Net framework directory. + /// + public bool ScanNetFrameworkDlls = true; + + /// + /// Whether to use mscorlib as a reference. + /// + public bool UseMscorlib = true; + + /// + /// Whether to search .csproj files. + /// + public bool AnalyseCsProjFiles = true; + + /// + /// The solution file to analyse, or null if not specified. + /// + public string SolutionFile; + + /// + /// Whether the extraction phase should be skipped (dry-run). + /// + public bool SkipExtraction = false; + + /// + /// Whether errors were encountered parsing the arguments. + /// + public bool Errors = false; + + /// + /// Whether to show help. + /// + public bool Help = false; + + /// + /// Determine whether the given path should be excluded. + /// + /// The path to query. + /// True iff the path matches an exclusion. + public bool ExcludesFile(string path) + { + return Excludes.Any(ex => path.Contains(ex)); + } + + /// + /// Outputs the command line options to the console. + /// + public void ShowHelp(System.IO.TextWriter output) + { + output.WriteLine("C# standalone extractor\n\nExtracts a C# project in the current directory without performing a build.\n"); + output.WriteLine("Additional options:\n"); + output.WriteLine(" xxx.sln Restrict sources to given solution"); + output.WriteLine(" --exclude:xxx Exclude a file or directory (can be specified multiple times)"); + output.WriteLine(" --references:xxx Scan additional files or directories for assemblies (can be specified multiple times)"); + output.WriteLine(" --skip-dotnet Do not reference the .Net Framework"); + output.WriteLine(" --dry-run Stop before extraction"); + output.WriteLine(" --skip-nuget Do not download nuget packages"); + output.WriteLine(" --all-references Use all references (default is to only use references in .csproj files)"); + output.WriteLine(" --nostdlib Do not link mscorlib.dll (use only for extracting mscorlib itself)"); + output.WriteLine(" --threads:nnn Specify number of threads (default=CPU cores)"); + output.WriteLine(" --verbose Produce more output"); + output.WriteLine(" --pdb Cross-reference information from PDBs where available"); + } + + private Options() + { + } + + public static Options Create(string[] args) + { + var options = new Options(); + options.ParseArguments(args); + return options; + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs new file mode 100644 index 00000000000..e0367fa63c1 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs @@ -0,0 +1,158 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Semmle.BuildAnalyser; +using Semmle.Util.Logging; + +namespace Semmle.Extraction.CSharp.Standalone +{ + /// + /// One independent run of the extractor. + /// + class Extraction + { + public Extraction(string directory) + { + this.directory = directory; + } + + public readonly string directory; + public readonly List Sources = new List(); + }; + + /// + /// Searches for source/references and creates separate extractions. + /// + class Analysis + { + readonly ILogger logger; + + public Analysis(ILogger logger) + { + this.logger = logger; + } + + // The extraction configuration for the entire project. + Extraction projectExtraction; + + public IEnumerable References + { + get; private set; + } + + /// + /// The extraction configuration. + /// + public Extraction Extraction => projectExtraction; + + /// + /// Creates an extraction for the current directory + /// and adds it to the list of all extractions. + /// + /// The directory of the extraction. + /// The extraction. + void CreateExtraction(string dir) + { + projectExtraction = new Extraction(dir); + } + + BuildAnalysis buildAnalysis; + + /// + /// Analyse projects/solution and resolves references. + /// + /// The build analysis options. + public void AnalyseProjects(Options options) + { + CreateExtraction(options.SrcDir); + var progressMonitor = new ProgressMonitor(logger); + buildAnalysis = new BuildAnalysis(options, progressMonitor); + References = buildAnalysis.ReferenceFiles; + projectExtraction.Sources.AddRange(options.SolutionFile == null ? buildAnalysis.AllSourceFiles : buildAnalysis.ProjectSourceFiles); + } + + /// + /// Delete any Nuget assemblies. + /// + public void Cleanup() + { + buildAnalysis.Cleanup(); + } + }; + + public class Program + { + static int Main(string[] args) + { + var options = Options.Create(args); + var output = new ConsoleLogger(options.Verbosity); + var a = new Analysis(output); + + if (options.Help) + { + options.ShowHelp(System.Console.Out); + return 0; + } + + if (options.Errors) + return 1; + + output.Log(Severity.Info, "Running C# standalone extractor"); + a.AnalyseProjects(options); + int sourceFiles = a.Extraction.Sources.Count(); + + if (sourceFiles == 0) + { + output.Log(Severity.Error, "No source files found"); + return 1; + } + + if (!options.SkipExtraction) + { + output.Log(Severity.Info, ""); + output.Log(Severity.Info, "Extracting..."); + Extractor.ExtractStandalone( + a.Extraction.Sources, + a.References, + new ExtractionProgress(output), + new FileLogger(options.Verbosity, Extractor.GetCSharpLogPath()), + options); + output.Log(Severity.Info, "Extraction complete"); + } + + a.Cleanup(); + return 0; + } + + class ExtractionProgress : IProgressMonitor + { + public ExtractionProgress(ILogger output) + { + logger = output; + } + + readonly ILogger logger; + + public void Analysed(int item, int total, string source, string output, TimeSpan time, AnalysisAction action) + { + logger.Log(Severity.Info, "[{0}/{1}] {2} ({3})", item, total, source, + action == AnalysisAction.Extracted ? time.ToString() : action == AnalysisAction.Excluded ? "excluded" : "up to date"); + } + + public void MissingType(string type) + { + logger.Log(Severity.Debug, "Missing type {0}", type); + } + + public void MissingNamespace(string @namespace) + { + logger.Log(Severity.Info, "Missing namespace {0}", @namespace); + } + + public void MissingSummary(int missingTypes, int missingNamespaces) + { + logger.Log(Severity.Info, "Failed to resolve {0} types and {1} namespaces", missingTypes, missingNamespaces); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs new file mode 100644 index 00000000000..f4bde55ec55 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs @@ -0,0 +1,105 @@ +using Semmle.Util.Logging; + +namespace Semmle.BuildAnalyser +{ + /// + /// Callback for various events that may happen during the build analysis. + /// + interface IProgressMonitor + { + void FindingFiles(string dir); + void UnresolvedReference(string id, string project); + void AnalysingProjectFiles(int count); + void FailedProjectFile(string filename, string reason); + void FailedNugetCommand(string exe, string args, string message); + void NugetInstall(string package); + void ResolvedReference(string filename); + void Summary(int existingSources, int usedSources, int missingSources, int references, int unresolvedReferences, int resolvedConflicts, int totalProjects, int failedProjects); + void Warning(string message); + void ResolvedConflict(string asm1, string asm2); + void MissingProject(string projectFile); + } + + class ProgressMonitor : IProgressMonitor + { + readonly ILogger logger; + + public ProgressMonitor(ILogger logger) + { + this.logger = logger; + } + + public void FindingFiles(string dir) + { + logger.Log(Severity.Info, "Finding files in {0}...", dir); + } + + public void IndexingReferences(int count) + { + logger.Log(Severity.Info, "Indexing..."); + logger.Log(Severity.Debug, "Indexing {0} DLLs...", count); + } + + public void UnresolvedReference(string id, string project) + { + logger.Log(Severity.Info, "Unresolved reference {0}", id); + logger.Log(Severity.Debug, "Unresolved {0} referenced by {1}", id, project); + } + + public void AnalysingProjectFiles(int count) + { + logger.Log(Severity.Info, "Analyzing project files..."); + } + + public void FailedProjectFile(string filename, string reason) + { + logger.Log(Severity.Info, "Couldn't read project file {0}: {1}", filename, reason); + } + + public void FailedNugetCommand(string exe, string args, string message) + { + logger.Log(Severity.Info, "Command failed: {0} {1}", exe, args); + logger.Log(Severity.Info, " {0}", message); + } + + public void NugetInstall(string package) + { + logger.Log(Severity.Info, "Restoring {0}...", package); + } + + public void ResolvedReference(string filename) + { + logger.Log(Severity.Info, "Resolved {0}", filename); + } + + public void Summary(int existingSources, int usedSources, int missingSources, + int references, int unresolvedReferences, int resolvedConflicts, int totalProjects, int failedProjects) + { + logger.Log(Severity.Info, ""); + logger.Log(Severity.Info, "Build analysis summary:"); + logger.Log(Severity.Info, "{0, 6} source files in the filesystem", existingSources); + logger.Log(Severity.Info, "{0, 6} source files in project files", usedSources); + logger.Log(Severity.Info, "{0, 6} sources missing from project files", missingSources); + logger.Log(Severity.Info, "{0, 6} resolved references", references); + logger.Log(Severity.Info, "{0, 6} unresolved references", unresolvedReferences); + logger.Log(Severity.Info, "{0, 6} resolved assembly conflicts", resolvedConflicts); + logger.Log(Severity.Info, "{0, 6} projects", totalProjects); + logger.Log(Severity.Info, "{0, 6} missing/failed projects", failedProjects); + } + + public void Warning(string message) + { + logger.Log(Severity.Warning, message); + } + + public void ResolvedConflict(string asm1, string asm2) + { + logger.Log(Severity.Info, "Resolved {0} as {1}", asm1, asm2); + } + + public void MissingProject(string projectFile) + { + logger.Log(Severity.Info, "Solution is missing {0}", projectFile); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Properties/AssemblyInfo.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Properties/AssemblyInfo.cs new file mode 100644 index 00000000000..4c5502fcde5 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Properties/AssemblyInfo.cs @@ -0,0 +1,35 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Semmle.Extraction.CSharp.Standalone")] +[assembly: AssemblyDescription("Standalone extractor for C#")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("Semmle Ltd.")] +[assembly: AssemblyProduct("Semmle.Extraction.CSharp.Standalone")] +[assembly: AssemblyCopyright("Copyright © Semmle 2016")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("bb71e9da-7e0a-43e8-989c-c8e87c828e7c")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Runtime.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Runtime.cs new file mode 100644 index 00000000000..c226f8dd6cb --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Runtime.cs @@ -0,0 +1,74 @@ +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.IO; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Standalone +{ + /// + /// Locates .NET Runtimes. + /// + static class Runtime + { + static string ExecutingRuntime => RuntimeEnvironment.GetRuntimeDirectory(); + + /// + /// Locates .NET Core Runtimes. + /// + public static IEnumerable CoreRuntimes + { + get + { + string[] dotnetDirs = { "/usr/share/dotnet", @"C:\Program Files\dotnet" }; + + foreach (var dir in dotnetDirs.Where(Directory.Exists)) + return Directory.EnumerateDirectories(Path.Combine(dir, "shared", "Microsoft.NETCore.App")). + OrderByDescending(d => Path.GetFileName(d)); + return Enumerable.Empty(); + } + } + + /// + /// Locates .NET Desktop Runtimes. + /// This includes Mono and Microsoft.NET. + /// + public static IEnumerable DesktopRuntimes + { + get + { + string[] monoDirs = { "/usr/lib/mono", @"C:\Program Files\Mono\lib\mono" }; + + if (Directory.Exists(@"C:\Windows\Microsoft.NET\Framework64")) + { + return System.IO.Directory.EnumerateDirectories(@"C:\Windows\Microsoft.NET\Framework64", "v*"). + OrderByDescending(d => Path.GetFileName(d)); + } + + foreach (var dir in monoDirs.Where(Directory.Exists)) + { + return System.IO.Directory.EnumerateDirectories(dir). + Where(d => Char.IsDigit(Path.GetFileName(d)[0])). + OrderByDescending(d => Path.GetFileName(d)); + } + + return Enumerable.Empty(); + } + } + + public static IEnumerable Runtimes + { + get + { + foreach (var r in CoreRuntimes) + yield return r; + + foreach (var r in DesktopRuntimes) + yield return r; + + // A bad choice if it's the self-contained runtime distributed in odasa dist. + yield return ExecutingRuntime; + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj new file mode 100644 index 00000000000..7b08c92c7db --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj @@ -0,0 +1,30 @@ + + + + Exe + netcoreapp2.0 + Semmle.Extraction.CSharp.Standalone + Semmle.Extraction.CSharp.Standalone + false + true + false + + + + + + + + + + + + + + + + + + + + diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/SolutionFile.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/SolutionFile.cs new file mode 100644 index 00000000000..b1a3edd4cf6 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/SolutionFile.cs @@ -0,0 +1,68 @@ +using Microsoft.Build.Construction; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace Semmle.BuildAnalyser +{ + /// + /// Access data in a .sln file. + /// + class SolutionFile + { + readonly Microsoft.Build.Construction.SolutionFile solutionFile; + + /// + /// Read the file. + /// + /// The filename of the .sln. + public SolutionFile(string filename) + { + // SolutionFile.Parse() expects a rooted path. + var fullPath = Path.GetFullPath(filename); + solutionFile = Microsoft.Build.Construction.SolutionFile.Parse(fullPath); + } + + /// + /// Projects directly included in the .sln file. + /// + public IEnumerable MsBuildProjects + { + get + { + return solutionFile.ProjectsInOrder. + Where(p => p.ProjectType == SolutionProjectType.KnownToBeMSBuildFormat). + Select(p => p.AbsolutePath). + Select(p => Path.DirectorySeparatorChar == '/' ? p.Replace("\\", "/") : p); + } + } + + /// + /// Projects included transitively via a subdirectory. + /// + public IEnumerable NestedProjects + { + get + { + return solutionFile.ProjectsInOrder. + Where(p => p.ProjectType == SolutionProjectType.SolutionFolder). + Where(p => Directory.Exists(p.AbsolutePath)). + SelectMany(p => new DirectoryInfo(p.AbsolutePath).EnumerateFiles("*.csproj", SearchOption.AllDirectories)). + Select(f => f.FullName); + } + } + + /// + /// List of projects which were mentioned but don't exist on disk. + /// + public IEnumerable MissingProjects => + // Only projects in the solution file can be missing. + // (NestedProjects are located on disk so always exist.) + MsBuildProjects.Where(p => !File.Exists(p)); + + /// + /// The list of project files. + /// + public IEnumerable Projects => MsBuildProjects.Concat(NestedProjects); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs b/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs new file mode 100644 index 00000000000..7b05a7bca0d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs @@ -0,0 +1,505 @@ +using System; +using System.Collections.Immutable; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.Diagnostics; +using System.IO; +using System.Linq; +using Semmle.Extraction.CSharp.Populators; +using System.Runtime.InteropServices; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; +using System.Diagnostics; +using Semmle.Util.Logging; +using Semmle.Util; + +namespace Semmle.Extraction.CSharp +{ + /// + /// Encapsulates a C# analysis task. + /// + public class Analyser : IDisposable + { + IExtractor extractor; + + readonly Stopwatch stopWatch = new Stopwatch(); + + readonly IProgressMonitor progressMonitor; + + public readonly ILogger Logger; + + public Analyser(IProgressMonitor pm, ILogger logger) + { + Logger = logger; + Logger.Log(Severity.Info, "EXTRACTION STARTING at {0}", DateTime.Now); + stopWatch.Start(); + progressMonitor = pm; + } + + CSharpCompilation compilation; + Layout layout; + + /// + /// Initialize the analyser. + /// + /// Arguments passed to csc. + /// The Roslyn compilation. + /// Extractor options. + public void Initialize( + CSharpCommandLineArguments commandLineArguments, + CSharpCompilation compilationIn, + Options options) + { + compilation = compilationIn; + + layout = new Layout(); + this.options = options; + + extractor = new Extraction.Extractor(false, GetOutputName(compilation, commandLineArguments), Logger); + + LogDiagnostics(); + SetReferencePaths(); + + CompilationErrors += FilteredDiagnostics.Count(); + } + + /// + /// Constructs the map from assembly string to its filename. + /// + /// Roslyn doesn't record the relationship between a filename and its assembly + /// information, so we need to retrieve this information manually. + /// + void SetReferencePaths() + { + foreach (var reference in compilation.References.OfType()) + { + try + { + var refPath = reference.FilePath; + + /* This method is significantly faster and more lightweight than using + * System.Reflection.Assembly.ReflectionOnlyLoadFrom. It is also allows + * loading the same assembly from different locations. + */ + using (var pereader = new System.Reflection.PortableExecutable.PEReader(new FileStream(refPath, FileMode.Open, FileAccess.Read, FileShare.Read))) + { + var metadata = pereader.GetMetadata(); + string assemblyIdentity; + unsafe + { + var reader = new System.Reflection.Metadata.MetadataReader(metadata.Pointer, metadata.Length); + var def = reader.GetAssemblyDefinition(); + assemblyIdentity = reader.GetString(def.Name) + " " + def.Version; + } + extractor.SetAssemblyFile(assemblyIdentity, refPath); + } + } + catch (Exception ex) + { + extractor.Message(new Message + { + exception = ex, + message = string.Format("Exception reading reference file {0}: {1}", + reference.FilePath, ex) + }); + } + } + } + + public void InitializeStandalone(CSharpCompilation compilationIn, CommonOptions options) + { + compilation = compilationIn; + layout = new Layout(); + extractor = new Extraction.Extractor(true, null, Logger); + this.options = options; + LogDiagnostics(); + SetReferencePaths(); + } + + readonly HashSet errorsToIgnore = new HashSet + { + "CS7027", // Code signing failure + "CS1589", // XML referencing not supported + "CS1569" // Error writing XML documentation + }; + + IEnumerable FilteredDiagnostics + { + get + { + return extractor == null || extractor.Standalone || compilation == null ? Enumerable.Empty() : + compilation. + GetDiagnostics(). + Where(e => e.Severity >= DiagnosticSeverity.Error && !errorsToIgnore.Contains(e.Id)); + } + } + + public IEnumerable MissingTypes => extractor.MissingTypes; + + public IEnumerable MissingNamespaces => extractor.MissingNamespaces; + + /// + /// Determine the path of the output dll/exe. + /// + /// Information about the compilation. + /// Cancellation token required. + /// The filename. + static string GetOutputName(CSharpCompilation compilation, + CSharpCommandLineArguments commandLineArguments) + { + // There's no apparent way to access the output filename from the compilation, + // so we need to re-parse the command line arguments. + + if (commandLineArguments.OutputFileName == null) + { + // No output specified: Use name based on first filename + var entry = compilation.GetEntryPoint(System.Threading.CancellationToken.None); + if (entry == null) + { + if (compilation.SyntaxTrees.Length == 0) + throw new ArgumentNullException("No source files seen"); + + // Probably invalid, but have a go anyway. + var entryPointFile = compilation.SyntaxTrees.First().FilePath; + return Path.ChangeExtension(entryPointFile, ".exe"); + } + else + { + var entryPointFilename = entry.Locations.First().SourceTree.FilePath; + return Path.ChangeExtension(entryPointFilename, ".exe"); + } + } + else + { + return Path.Combine(commandLineArguments.OutputDirectory, commandLineArguments.OutputFileName); + } + } + + /// + /// Perform an analysis on a source file/syntax tree. + /// + /// Syntax tree to analyse. + public void AnalyseTree(SyntaxTree tree) + { + extractionTasks.Add(() => DoExtractTree(tree)); + } + + /// + /// Perform an analysis on an assembly. + /// + /// Assembly to analyse. + void AnalyseAssembly(PortableExecutableReference assembly) + { + // CIL first - it takes longer. + if (options.CIL) + extractionTasks.Add(() => DoExtractCIL(assembly)); + extractionTasks.Add(() => DoAnalyseAssembly(assembly)); + } + + readonly object progressMutex = new object(); + int taskCount = 0; + + CommonOptions options; + + static bool FileIsUpToDate(string src, string dest) + { + return File.Exists(dest) && + File.GetLastWriteTime(dest) >= File.GetLastWriteTime(src); + } + + bool FileIsCached(string src, string dest) + { + return options.Cache && FileIsUpToDate(src, dest); + } + + /// + /// Extract an assembly to a new trap file. + /// If the trap file exists, skip extraction to avoid duplicating + /// extraction within the snapshot. + /// + /// The assembly to extract. + void DoAnalyseAssembly(PortableExecutableReference r) + { + try + { + var stopwatch = new Stopwatch(); + stopwatch.Start(); + + var assemblyPath = r.FilePath; + var projectLayout = layout.LookupProjectOrDefault(assemblyPath); + using (var trapWriter = projectLayout.CreateTrapWriter(Logger, assemblyPath, true)) + { + var skipExtraction = FileIsCached(assemblyPath, trapWriter.TrapFile); + + if (!skipExtraction) + { + /* Note on parallel builds: + * + * The trap writer and source archiver both perform atomic moves + * of the file to the final destination. + * + * If the same source file or trap file are generated concurrently + * (by different parallel invocations of the extractor), then + * last one wins. + * + * Specifically, if two assemblies are analysed concurrently in a build, + * then there is a small amount of duplicated work but the output should + * still be correct. + */ + + // compilation.Clone() reduces memory footprint by allowing the symbols + // in c to be garbage collected. + Compilation c = compilation.Clone(); + + var assembly = c.GetAssemblyOrModuleSymbol(r) as IAssemblySymbol; + + if (assembly != null) + { + var cx = new Context(extractor, c, trapWriter, new AssemblyScope(assembly, assemblyPath)); + + foreach (var module in assembly.Modules) + { + AnalyseNamespace(cx, module.GlobalNamespace); + } + + cx.PopulateAll(); + } + } + + ReportProgress(assemblyPath, trapWriter.TrapFile, stopwatch.Elapsed, skipExtraction ? AnalysisAction.UpToDate : AnalysisAction.Extracted); + } + } + catch (Exception ex) + { + Logger.Log(Severity.Error, " Unhandled exception analyzing {0}: {1}", r.FilePath, ex); + } + } + + void DoExtractCIL(PortableExecutableReference r) + { + var stopwatch = new Stopwatch(); + stopwatch.Start(); + string trapFile; + bool extracted; + CIL.Entities.Assembly.ExtractCIL(layout, r.FilePath, Logger, !options.Cache, options.PDB, out trapFile, out extracted); + stopwatch.Stop(); + ReportProgress(r.FilePath, trapFile, stopwatch.Elapsed, extracted ? AnalysisAction.Extracted : AnalysisAction.UpToDate); + } + + void AnalyseNamespace(Context cx, INamespaceSymbol ns) + { + foreach (var memberNamespace in ns.GetNamespaceMembers()) + { + AnalyseNamespace(cx, memberNamespace); + } + + foreach (var memberType in ns.GetTypeMembers()) + { + Entities.Type.Create(cx, memberType).ExtractRecursive(); + } + } + + /// + /// Enqueue all reference analysis tasks. + /// + public void AnalyseReferences() + { + foreach (var r in compilation.References.OfType()) + { + AnalyseAssembly(r); + } + } + + // The bulk of the extraction work, potentially executed in parallel. + readonly List extractionTasks = new List(); + + void ReportProgress(string src, string output, TimeSpan time, AnalysisAction action) + { + lock (progressMutex) + progressMonitor.Analysed(++taskCount, extractionTasks.Count, src, output, time, action); + } + + void DoExtractTree(SyntaxTree tree) + { + try + { + var stopwatch = new Stopwatch(); + stopwatch.Start(); + var sourcePath = tree.FilePath; + + var projectLayout = layout.LookupProjectOrNull(sourcePath); + bool excluded = projectLayout == null; + string trapPath = excluded ? "" : projectLayout.GetTrapPath(Logger, sourcePath); + bool upToDate = false; + + if (!excluded) + { + // compilation.Clone() is used to allow symbols to be garbage collected. + using (var trapWriter = projectLayout.CreateTrapWriter(Logger, sourcePath, false)) + { + upToDate = options.Fast && FileIsUpToDate(sourcePath, trapWriter.TrapFile); + + if (!upToDate) + { + Context cx = new Context(extractor, compilation.Clone(), trapWriter, new SourceScope(tree)); + Populators.CompilationUnit.Extract(cx, tree.GetRoot()); + cx.PopulateAll(); + cx.ExtractComments(cx.CommentGenerator); + } + } + } + + ReportProgress(sourcePath, trapPath, stopwatch.Elapsed, excluded ? AnalysisAction.Excluded : upToDate ? AnalysisAction.UpToDate : AnalysisAction.Extracted); + } + catch (Exception ex) + { + extractor.Message(new Message { exception = ex, message = string.Format("Unhandled exception processing {0}: {1}", tree.FilePath, ex), severity = Severity.Error }); + } + } + + /// + /// Run all extraction tasks. + /// + /// The number of threads to use. + public void PerformExtraction(int numberOfThreads) + { + Parallel.Invoke( + new ParallelOptions { MaxDegreeOfParallelism = numberOfThreads }, + extractionTasks.ToArray()); + } + + public void Dispose() + { + stopWatch.Stop(); + Logger.Log(Severity.Info, " Peak working set = {0} MB", Process.GetCurrentProcess().PeakWorkingSet64 / (1024 * 1024)); + + if (TotalErrors > 0) + Logger.Log(Severity.Info, "EXTRACTION FAILED with {0} error{1} in {2}", TotalErrors, TotalErrors == 1 ? "" : "s", stopWatch.Elapsed); + else + Logger.Log(Severity.Info, "EXTRACTION SUCCEEDED in {0}", stopWatch.Elapsed); + + Logger.Dispose(); + } + + /// + /// Number of errors encountered during extraction. + /// + public int ExtractorErrors => extractor == null ? 0 : extractor.Errors; + + /// + /// Number of errors encountered by the compiler. + /// + public int CompilationErrors { get; set; } + + /// + /// Total number of errors reported. + /// + public int TotalErrors => CompilationErrors + ExtractorErrors; + + void AppendQuoted(StringBuilder sb, string s) + { + if (s.IndexOf(' ') != -1) + sb.Append('\"').Append(s).Append('\"'); + else + sb.Append(s); + } + + /// + /// Logs detailed information about this invocation, + /// in the event that errors were detected. + /// + public void LogDiagnostics() + { + Logger.Log(Severity.Info, " Current working directory: {0}", Directory.GetCurrentDirectory()); + Logger.Log(Severity.Info, " Extractor: {0}", Environment.GetCommandLineArgs().First()); + if (extractor != null) + Logger.Log(Severity.Info, " Extractor version: {0}", extractor.Version); + var sb = new StringBuilder(); + sb.Append(" Expanded command line: "); + bool first = true; + foreach (var arg in Environment.GetCommandLineArgs().Skip(1)) + { + if (arg[0] == '@') + { + foreach (var line in File.ReadAllLines(arg.Substring(1))) + { + if (first) first = false; + else sb.Append(" "); + sb.Append(line); + } + } + else + { + if (first) first = false; + else sb.Append(" "); + AppendQuoted(sb, arg); + } + } + Logger.Log(Severity.Info, sb.ToString()); + + foreach (var error in FilteredDiagnostics) + { + Logger.Log(Severity.Error, " Compilation error: {0}", error); + } + + if (FilteredDiagnostics.Any()) + { + foreach (var reference in compilation.References) + { + Logger.Log(Severity.Info, " Resolved reference {0}", reference.Display); + } + } + } + } + + /// + /// What action was performed when extracting a file. + /// + public enum AnalysisAction + { + Extracted, + UpToDate, + Excluded + } + + /// + /// Callback for various extraction events. + /// (Used for display of progress). + /// + public interface IProgressMonitor + { + /// + /// Callback that a particular item has been analysed. + /// + /// The item number being processed. + /// The total number of items to process. + /// The name of the item, e.g. a source file. + /// The name of the item being output, e.g. a trap file. + /// The time to extract the item. + /// What action was taken for the file. + void Analysed(int item, int total, string source, string output, TimeSpan time, AnalysisAction action); + + /// + /// A "using namespace" directive was seen but the given + /// namespace could not be found. + /// Only called once for each @namespace. + /// + /// + void MissingNamespace(string @namespace); + + /// + /// An ErrorType was found. + /// Called once for each type name. + /// + /// The full/partial name of the type. + void MissingType(string type); + + /// + /// Report a summary of missing entities. + /// + /// The number of missing types. + /// The number of missing using namespace declarations. + void MissingSummary(int types, int namespaces); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/CompilerVersion.cs b/csharp/extractor/Semmle.Extraction.CSharp/CompilerVersion.cs new file mode 100644 index 00000000000..ee1fca5bc2d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/CompilerVersion.cs @@ -0,0 +1,107 @@ +using System.Diagnostics; +using System.IO; +using System.Runtime.InteropServices; + +namespace Semmle.Extraction.CSharp +{ + /// + /// Identifies the compiler and framework from the command line arguments. + /// --compiler specifies the compiler + /// --framework specifies the .net framework + /// + public class CompilerVersion + { + const string csc_rsp = "csc.rsp"; + readonly string specifiedFramework = null; + + /// + /// The value specified by --compiler, or null. + /// + public string SpecifiedCompiler + { + get; + private set; + } + + /// + /// Why was the candidate exe rejected as a compiler? + /// + public string SkipReason + { + get; + private set; + } + + /// + /// Probes the compiler (if specified). + /// + /// The command line arguments. + public CompilerVersion(Options options) + { + SpecifiedCompiler = options.CompilerName; + specifiedFramework = options.Framework; + + if (SpecifiedCompiler != null) + { + if (!File.Exists(SpecifiedCompiler)) + { + SkipExtractionBecause("the specified file does not exist"); + return; + } + + // Reads the file details from the .exe + var versionInfo = FileVersionInfo.GetVersionInfo(SpecifiedCompiler); + + var compilerDir = Path.GetDirectoryName(SpecifiedCompiler); + bool known_compiler_name = versionInfo.OriginalFilename == "csc.exe" || versionInfo.OriginalFilename == "csc2.exe"; + bool copyright_microsoft = versionInfo.LegalCopyright != null && versionInfo.LegalCopyright.Contains("Microsoft"); + bool mscorlib_exists = File.Exists(Path.Combine(compilerDir, "mscorlib.dll")); + + if (specifiedFramework == null && mscorlib_exists) + { + specifiedFramework = compilerDir; + } + + if (!known_compiler_name) + { + SkipExtractionBecause("the exe name is not recognised"); + } + else if (!copyright_microsoft) + { + SkipExtractionBecause("the exe isn't copyright Microsoft"); + } + } + } + + void SkipExtractionBecause(string reason) + { + SkipExtraction = true; + SkipReason = reason; + } + + /// + /// The directory containing the .Net Framework. + /// + public string FrameworkPath => specifiedFramework ?? RuntimeEnvironment.GetRuntimeDirectory(); + + /// + /// The file csc.rsp. + /// + public string CscRsp => Path.Combine(FrameworkPath, csc_rsp); + + /// + /// Should we skip extraction? + /// Only if csc.exe was specified but it wasn't a compiler. + /// + public bool SkipExtraction + { + get; + private set; + } + + /// + /// Gets additional reference directories - the compiler directory. + /// + public string AdditionalReferenceDirectories => SpecifiedCompiler != null ? Path.GetDirectoryName(SpecifiedCompiler) : null; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Accessor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Accessor.cs new file mode 100644 index 00000000000..b70ab6ce086 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Accessor.cs @@ -0,0 +1,88 @@ +using Microsoft.CodeAnalysis; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class Accessor : Method + { + protected Accessor(Context cx, IMethodSymbol init) + : base(cx, init) { } + + /// + /// Gets the property symbol associated with this accessor. + /// + IPropertySymbol PropertySymbol + { + get + { + // Usually, the property/indexer can be fetched from the associated symbol + var prop = symbol.AssociatedSymbol as IPropertySymbol; + if (prop != null) + return prop; + + // But for properties/indexers that implement explicit interfaces, Roslyn + // does not properly populate `AssociatedSymbol` + var props = symbol.ContainingType.GetMembers().OfType(); + props = props.Where(p => symbol.Equals(p.GetMethod) || symbol.Equals(p.SetMethod)); + return props.SingleOrDefault(); + } + } + + public new Accessor OriginalDefinition => Create(Context, symbol.OriginalDefinition); + + public override void Populate() + { + PopulateMethod(); + ExtractModifiers(); + ContainingType.ExtractGenerics(); + + var prop = PropertySymbol; + if (prop == null) + { + Context.ModelError(symbol, "Unhandled accessor associated symbol"); + return; + } + + var parent = Property.Create(Context, prop); + int kind; + Accessor unboundAccessor; + if (symbol.Equals(prop.GetMethod)) + { + kind = 1; + unboundAccessor = Create(Context, prop.OriginalDefinition.GetMethod); + } + else if (symbol.Equals(prop.SetMethod)) + { + kind = 2; + unboundAccessor = Create(Context, prop.OriginalDefinition.SetMethod); + } + else + { + Context.ModelError(symbol, "Undhandled accessor kind"); + return; + } + + Context.Emit(Tuples.accessors(this, kind, symbol.Name, parent, unboundAccessor)); + + foreach (var l in Locations) + Context.Emit(Tuples.accessor_location(this, l)); + + Overrides(); + + if (symbol.FromSource() && Block == null) + { + Context.Emit(Tuples.compiler_generated(this)); + } + } + + public new static Accessor Create(Context cx, IMethodSymbol symbol) => + AccessorFactory.Instance.CreateEntity(cx, symbol); + + class AccessorFactory : ICachedEntityFactory + { + public static readonly AccessorFactory Instance = new AccessorFactory(); + + public Accessor Create(Context cx, IMethodSymbol init) => new Accessor(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs new file mode 100644 index 00000000000..41deadc7fe9 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs @@ -0,0 +1,79 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Entities; +using System.Collections.Generic; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class Attribute : FreshEntity, IExpressionParentEntity + { + bool IExpressionParentEntity.IsTopLevelParent => true; + + public Attribute(Context cx, AttributeData attribute, IEntity entity) + : base(cx) + { + if (attribute.ApplicationSyntaxReference != null) + { + // !! Extract attributes from assemblies. + // This is harder because the "expression" entities presume the + // existence of a syntax tree. This is not the case for compiled + // attributes. + var syntax = attribute.ApplicationSyntaxReference.GetSyntax() as AttributeSyntax; + ExtractAttribute(cx, syntax, attribute.AttributeClass, entity); + } + } + + public Attribute(Context cx, AttributeSyntax attribute, IEntity entity) + : base(cx) + { + var info = cx.GetSymbolInfo(attribute); + ExtractAttribute(cx, attribute, info.Symbol.ContainingType, entity); + } + + void ExtractAttribute(Context cx, AttributeSyntax syntax, ITypeSymbol attributeClass, IEntity entity) + { + var type = Type.Create(cx, attributeClass); + cx.Emit(Tuples.attributes(this, type.TypeRef, entity)); + + cx.Emit(Tuples.attribute_location(this, cx.Create(syntax.Name.GetLocation()))); + + if (cx.Extractor.OutputPath != null) + cx.Emit(Tuples.attribute_location(this, Assembly.CreateOutputAssembly(cx))); + + TypeMention.Create(cx, syntax.Name, this, type); + + if (syntax.ArgumentList != null) + { + cx.PopulateLater(() => + { + int child = 0; + foreach (var arg in syntax.ArgumentList.Arguments) + { + Expression.Create(cx, arg.Expression, this, child++); + } + // !! Handle named arguments + }); + } + } + + public static void ExtractAttributes(Context cx, ISymbol symbol, IEntity entity) + { + foreach (var attribute in symbol.GetAttributes()) + { + new Attribute(cx, attribute, entity); + } + } + + public static void ExtractAttributes(Context cx, IEnumerable attributes, IEntity entity) + { + foreach (var attributeSyntax in attributes.SelectMany(l => l.Attributes)) + { + new Attribute(cx, attributeSyntax, entity); + } + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.OptionalLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentBlock.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentBlock.cs new file mode 100644 index 00000000000..990b287cd70 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentBlock.cs @@ -0,0 +1,51 @@ +using Semmle.Extraction.CommentProcessing; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities +{ + class CommentBlock : CachedEntity + { + CommentBlock(Context cx, ICommentBlock init) + : base(cx, init) { } + + public override void Populate() + { + Context.Emit(Tuples.commentblock(this)); + int child = 0; + Context.Emit(Tuples.commentblock_location(this, Context.Create(symbol.Location))); + foreach (var l in symbol.CommentLines) + { + Context.Emit(Tuples.commentblock_child(this, (CommentLine)l, child++)); + } + } + + public override bool NeedsPopulation => true; + + public override IId Id + { + get + { + var loc = Context.Create(symbol.Location); + return new Key(loc, ";commentblock"); + } + } + + public override Microsoft.CodeAnalysis.Location ReportingLocation => symbol.Location; + + public void BindTo(Label entity, Binding binding) + { + Context.Emit(Tuples.commentblock_binding(this, entity, binding)); + } + + public static CommentBlock Create(Context cx, ICommentBlock block) => CommentBlockFactory.Instance.CreateEntity(cx, block); + + class CommentBlockFactory : ICachedEntityFactory + { + public static readonly CommentBlockFactory Instance = new CommentBlockFactory(); + + public CommentBlock Create(Context cx, ICommentBlock init) => new CommentBlock(cx, init); + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs new file mode 100644 index 00000000000..234ff96935e --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs @@ -0,0 +1,147 @@ +using Semmle.Extraction.CommentProcessing; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Semmle.Extraction.Entities; +using System; + +namespace Semmle.Extraction.CSharp.Entities +{ + class CommentLine : CachedEntity<(Microsoft.CodeAnalysis.Location, string)>, ICommentLine + { + CommentLine(Context cx, Microsoft.CodeAnalysis.Location loc, CommentType type, string text, string raw) + : base(cx, (loc, text)) + { + Type = type; + RawText = raw; + } + + public Microsoft.CodeAnalysis.Location Location => symbol.Item1; + public CommentType Type { get; private set; } + + public string Text { get { return symbol.Item2; } } + public string RawText { get; private set; } + + public static void Extract(Context cx, SyntaxTrivia trivia) + { + switch (trivia.Kind()) + { + case SyntaxKind.SingleLineDocumentationCommentTrivia: + /* + This is actually a multi-line comment consisting of /// lines. + So split it up. + */ + + var text = trivia.ToFullString(); + + var split = text.Split('\n'); + var currentLocation = trivia.GetLocation().SourceSpan.Start - 3; + + for (int line = 0; line < split.Length - 1; ++line) + { + string fullLine = split[line]; + var nextLineLocation = currentLocation + fullLine.Length + 1; + fullLine = fullLine.TrimEnd('\r'); + string trimmedLine = fullLine; + + int leadingSpaces = trimmedLine.IndexOf('/'); + if (leadingSpaces != -1) + { + fullLine = fullLine.Substring(leadingSpaces); + currentLocation += leadingSpaces; + trimmedLine = trimmedLine.Substring(leadingSpaces + 3); // Remove leading spaces and the "///" + trimmedLine = trimmedLine.Trim(); + + var span = Microsoft.CodeAnalysis.Text.TextSpan.FromBounds(currentLocation, currentLocation + fullLine.Length); + var location = Microsoft.CodeAnalysis.Location.Create(trivia.SyntaxTree, span); + var commentType = CommentType.XmlDoc; + cx.CommentGenerator.AddComment(Create(cx, location, commentType, trimmedLine, fullLine)); + } + else + { + cx.ModelError("Unexpected comment format"); + } + currentLocation = nextLineLocation; + } + break; + + case SyntaxKind.SingleLineCommentTrivia: + { + string contents = trivia.ToString().Substring(2); + var commentType = CommentType.Singleline; + if (contents.Length > 0 && contents[0] == '/') + { + commentType = CommentType.XmlDoc; + contents = contents.Substring(1); // An XML comment. + } + cx.CommentGenerator.AddComment(Create(cx, trivia.GetLocation(), commentType, contents.Trim(), trivia.ToFullString())); + } + break; + case SyntaxKind.MultiLineDocumentationCommentTrivia: + case SyntaxKind.MultiLineCommentTrivia: + /* We receive a single SyntaxTrivia for a multiline block spanning several lines. + So we split it into separate lines + */ + text = trivia.ToFullString(); + + split = text.Split('\n'); + currentLocation = trivia.GetLocation().SourceSpan.Start; + + for (int line = 0; line < split.Length; ++line) + { + string fullLine = split[line]; + var nextLineLocation = currentLocation + fullLine.Length + 1; + fullLine = fullLine.TrimEnd('\r'); + string trimmedLine = fullLine; + if (line == 0) trimmedLine = trimmedLine.Substring(2); + if (line == split.Length - 1) trimmedLine = trimmedLine.Substring(0, trimmedLine.Length - 2); + trimmedLine = trimmedLine.Trim(); + + var span = Microsoft.CodeAnalysis.Text.TextSpan.FromBounds(currentLocation, currentLocation + fullLine.Length); + var location = Microsoft.CodeAnalysis.Location.Create(trivia.SyntaxTree, span); + var commentType = line == 0 ? CommentType.Multiline : CommentType.MultilineContinuation; + cx.CommentGenerator.AddComment(Create(cx, location, commentType, trimmedLine, fullLine)); + currentLocation = nextLineLocation; + } + break; + // Strangely, these are reported as SingleLineCommentTrivia. + case SyntaxKind.DocumentationCommentExteriorTrivia: + cx.ModelError("Unhandled comment type {0} for {1}", trivia.Kind(), trivia); + break; + } + } + + Extraction.Entities.Location location; + + public override void Populate() + { + location = Context.Create(Location); + Context.Emit(Tuples.commentline(this, Type == CommentType.MultilineContinuation ? CommentType.Multiline : Type, Text, RawText)); + Context.Emit(Tuples.commentline_location(this, location)); + } + + public override Microsoft.CodeAnalysis.Location ReportingLocation => location.symbol; + + public override bool NeedsPopulation => true; + + public override IId Id + { + get + { + var loc = Context.Create(Location); + return new Key(loc, ";commentline"); + } + } + + static CommentLine Create(Context cx, Microsoft.CodeAnalysis.Location loc, CommentType type, string text, string raw) => CommentLineFactory.Instance.CreateEntity(cx, loc, type, text, raw); + + class CommentLineFactory : ICachedEntityFactory<(Microsoft.CodeAnalysis.Location, CommentType, string, string), CommentLine> + { + public static readonly CommentLineFactory Instance = new CommentLineFactory(); + + public CommentLine Create(Context cx, (Microsoft.CodeAnalysis.Location, CommentType, string, string) init) => + new CommentLine(cx, init.Item1, init.Item2, init.Item3, init.Item4); + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.OptionalLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Constructor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Constructor.cs new file mode 100644 index 00000000000..d96db4b4624 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Constructor.cs @@ -0,0 +1,158 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Util; +using System.Linq; +using Microsoft.CodeAnalysis.CSharp; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities +{ + public class Constructor : Method + { + Constructor(Context cx, IMethodSymbol init) + : base(cx, init) { } + + public override void Populate() + { + PopulateMethod(); + ExtractModifiers(); + ContainingType.ExtractGenerics(); + + Context.Emit(Tuples.constructors(this, symbol.ContainingType.Name, ContainingType, (Constructor)OriginalDefinition)); + Context.Emit(Tuples.constructor_location(this, Location)); + + if (symbol.IsImplicitlyDeclared) + { + var lineCounts = new LineCounts() { Total = 2, Code = 1, Comment = 0 }; + Context.Emit(Tuples.numlines(this, lineCounts)); + } + ExtractCompilerGenerated(); + } + + protected override void ExtractInitializers() + { + // Do not extract initializers for constructed types. + if (!IsSourceDeclaration) return; + + var syntax = Syntax; + var initializer = syntax == null ? null : syntax.Initializer; + + if (initializer == null) return; + + Type initializerType; + var symbolInfo = Context.GetSymbolInfo(initializer); + + switch (initializer.Kind()) + { + case SyntaxKind.BaseConstructorInitializer: + initializerType = Type.Create(Context, symbol.ContainingType.BaseType); + break; + case SyntaxKind.ThisConstructorInitializer: + initializerType = ContainingType; + break; + default: + Context.ModelError(initializer, "Unknown initializer"); + return; + } + + var initInfo = new ExpressionInfo(Context, + initializerType, + Context.Create(initializer.ThisOrBaseKeyword.GetLocation()), + Kinds.ExprKind.CONSTRUCTOR_INIT, + this, + -1, + false, + null); + + var init = new Expression(initInfo); + + var target = Constructor.Create(Context, (IMethodSymbol)symbolInfo.Symbol); + + if (target == null) + { + Context.ModelError(symbol, "Unable to resolve call"); + return; + } + + Context.Emit(Tuples.expr_call(init, target)); + + int child = 0; + foreach (var arg in initializer.ArgumentList.Arguments) + { + Expression.Create(Context, arg.Expression, init, child++); + } + } + + ConstructorDeclarationSyntax Syntax + { + get + { + return symbol.DeclaringSyntaxReferences. + Select(r => r.GetSyntax()). + OfType(). + FirstOrDefault(); + } + } + + public new static Constructor Create(Context cx, IMethodSymbol constructor) + { + if (constructor == null) return null; + + switch (constructor.MethodKind) + { + case MethodKind.StaticConstructor: + case MethodKind.Constructor: + return ConstructorFactory.Instance.CreateEntity(cx, constructor); + default: + throw new InternalError(constructor, "Attempt to create a Constructor from a symbol that isn't a constructor"); + } + } + + public override IId Id + { + get + { + return new Key(tb => + { + if (symbol.IsStatic) tb.Append("static"); + tb.Append(ContainingType); + AddParametersToId(Context, tb, symbol); + tb.Append("; constructor"); + }); + } + } + + ConstructorDeclarationSyntax GetSyntax() => + symbol.DeclaringSyntaxReferences.Select(r => r.GetSyntax()).OfType().FirstOrDefault(); + + public override Microsoft.CodeAnalysis.Location FullLocation => ReportingLocation; + + public override Microsoft.CodeAnalysis.Location ReportingLocation + { + get + { + var syn = GetSyntax(); + if (syn != null) + { + return syn.Identifier.GetLocation(); + } + else if (symbol.IsImplicitlyDeclared) + { + return ContainingType.ReportingLocation; + } + else + { + return symbol.ContainingType.Locations.FirstOrDefault(); + } + } + } + + class ConstructorFactory : ICachedEntityFactory + { + public static readonly ConstructorFactory Instance = new ConstructorFactory(); + + public Constructor Create(Context cx, IMethodSymbol init) => new Constructor(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Conversion.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Conversion.cs new file mode 100644 index 00000000000..e365ca53ae0 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Conversion.cs @@ -0,0 +1,37 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class Conversion : UserOperator + { + Conversion(Context cx, IMethodSymbol init) + : base(cx, init) { } + + public new static Conversion Create(Context cx, IMethodSymbol symbol) => + ConversionFactory.Instance.CreateEntity(cx, symbol); + + public override Microsoft.CodeAnalysis.Location ReportingLocation + { + get + { + return symbol. + DeclaringSyntaxReferences. + Select(r => r.GetSyntax()). + OfType(). + Select(s => s.FixedLocation()). + Concat(symbol.Locations). + FirstOrDefault(); + } + } + + class ConversionFactory : ICachedEntityFactory + { + public static readonly ConversionFactory Instance = new ConversionFactory(); + + public Conversion Create(Context cx, IMethodSymbol init) => new Conversion(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Destructor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Destructor.cs new file mode 100644 index 00000000000..5a4b9093351 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Destructor.cs @@ -0,0 +1,35 @@ +using Microsoft.CodeAnalysis; + +namespace Semmle.Extraction.CSharp.Entities +{ + class Destructor : Method + { + Destructor(Context cx, IMethodSymbol init) + : base(cx, init) { } + + public override void Populate() + { + PopulateMethod(); + ExtractModifiers(); + ContainingType.ExtractGenerics(); + + Context.Emit(Tuples.destructors(this, string.Format("~{0}", symbol.ContainingType.Name), ContainingType, OriginalDefinition(Context, this, symbol))); + Context.Emit(Tuples.destructor_location(this, Location)); + } + + static new Destructor OriginalDefinition(Context cx, Destructor original, IMethodSymbol symbol) + { + return symbol.OriginalDefinition == null || Equals(symbol.OriginalDefinition, symbol) ? original : Create(cx, symbol.OriginalDefinition); + } + + public new static Destructor Create(Context cx, IMethodSymbol symbol) => + DestructorFactory.Instance.CreateEntity(cx, symbol); + + class DestructorFactory : ICachedEntityFactory + { + public static readonly DestructorFactory Instance = new DestructorFactory(); + + public Destructor Create(Context cx, IMethodSymbol init) => new Destructor(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Event.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Event.cs new file mode 100644 index 00000000000..86a22c704a7 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Event.cs @@ -0,0 +1,76 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class Event : CachedSymbol + { + Event(Context cx, IEventSymbol init) + : base(cx, init) { } + + public override IId Id + { + get + { + return new Key(tb => + { + tb.Append(ContainingType); + tb.Append("."); + Method.AddExplicitInterfaceQualifierToId(Context, tb, symbol.ExplicitInterfaceImplementations); + tb.Append(symbol.Name); + tb.Append(";event"); + }); + } + } + + public override void Populate() + { + var type = Type.Create(Context, symbol.Type); + Context.Emit(Tuples.events(this, symbol.GetName(), ContainingType, type.TypeRef, Create(Context, symbol.OriginalDefinition))); + + var adder = symbol.AddMethod; + if (adder != null) + EventAccessor.Create(Context, adder); + + var remover = symbol.RemoveMethod; + if (remover != null) + EventAccessor.Create(Context, remover); + + ExtractModifiers(); + BindComments(); + + var declSyntaxReferences = IsSourceDeclaration + ? symbol.DeclaringSyntaxReferences.Select(d => d.GetSyntax()).ToArray() + : Enumerable.Empty(); + + foreach (var explicitInterface in symbol.ExplicitInterfaceImplementations.Select(impl => Type.Create(Context, impl.ContainingType))) + { + Context.Emit(Tuples.explicitly_implements(this, explicitInterface.TypeRef)); + + foreach (var syntax in declSyntaxReferences.OfType()) + TypeMention.Create(Context, syntax.ExplicitInterfaceSpecifier.Name, this, explicitInterface); + } + + foreach (var l in Locations) + Context.Emit(Tuples.event_location(this, l)); + + foreach (var syntaxType in declSyntaxReferences.OfType(). + Select(d => d.Parent). + OfType(). + Select(syntax => syntax.Type)) + TypeMention.Create(Context, syntaxType, this, type); + } + + public static Event Create(Context cx, IEventSymbol symbol) => EventFactory.Instance.CreateEntity(cx, symbol); + + class EventFactory : ICachedEntityFactory + { + public static readonly EventFactory Instance = new EventFactory(); + + public Event Create(Context cx, IEventSymbol init) => new Event(cx, init); + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/EventAccessor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/EventAccessor.cs new file mode 100644 index 00000000000..7e6cb01a8b6 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/EventAccessor.cs @@ -0,0 +1,64 @@ +using Microsoft.CodeAnalysis; + +namespace Semmle.Extraction.CSharp.Entities +{ + class EventAccessor : Accessor + { + EventAccessor(Context cx, IMethodSymbol init) + : base(cx, init) { } + + /// + /// Gets the event symbol associated with this accessor. + /// + IEventSymbol EventSymbol => symbol.AssociatedSymbol as IEventSymbol; + + public override void Populate() + { + PopulateMethod(); + ContainingType.ExtractGenerics(); + + var @event = EventSymbol; + if (@event == null) + { + Context.ModelError(symbol, "Unhandled event accessor associated symbol"); + return; + } + + var parent = Event.Create(Context, @event); + int kind; + EventAccessor unboundAccessor; + if (symbol.Equals(@event.AddMethod)) + { + kind = 1; + unboundAccessor = Create(Context, @event.OriginalDefinition.AddMethod); + } + else if (symbol.Equals(@event.RemoveMethod)) + { + kind = 2; + unboundAccessor = Create(Context, @event.OriginalDefinition.RemoveMethod); + } + else + { + Context.ModelError(symbol, "Undhandled event accessor kind"); + return; + } + + Context.Emit(Tuples.event_accessors(this, kind, symbol.Name, parent, unboundAccessor)); + + foreach (var l in Locations) + Context.Emit(Tuples.event_accessor_location(this, l)); + + Overrides(); + } + + public new static EventAccessor Create(Context cx, IMethodSymbol symbol) => + EventAccessorFactory.Instance.CreateEntity(cx, symbol); + + class EventAccessorFactory : ICachedEntityFactory + { + public static readonly EventAccessorFactory Instance = new EventAccessorFactory(); + + public EventAccessor Create(Context cx, IMethodSymbol init) => new EventAccessor(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expression.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expression.cs new file mode 100644 index 00000000000..f52e241f031 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expression.cs @@ -0,0 +1,527 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Entities; +using Semmle.Extraction.Kinds; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + public interface IExpressionParentEntity : IEntity + { + /// + /// Whether this entity is the parent of a top-level expression. + /// + bool IsTopLevelParent { get; } + } + + class Expression : FreshEntity, IExpressionParentEntity + { + public readonly Type Type; + public readonly Extraction.Entities.Location Location; + public readonly ExprKind Kind; + + internal Expression(IExpressionInfo info) + : base(info.Context) + { + Location = info.Location; + Kind = info.Kind; + Type = info.Type ?? NullType.Create(cx); + + cx.Emit(Tuples.expressions(this, Kind, Type.TypeRef)); + if (info.Parent.IsTopLevelParent) + cx.Emit(Tuples.expr_parent_top_level(this, info.Child, info.Parent)); + else + cx.Emit(Tuples.expr_parent(this, info.Child, info.Parent)); + cx.Emit(Tuples.expr_location(this, Location)); + + if (info.IsCompilerGenerated) + cx.Emit(Tuples.expr_compiler_generated(this)); + + if (info.ExprValue is string value) + cx.Emit(Tuples.expr_value(this, value)); + + Type.ExtractGenerics(); + } + + public override Microsoft.CodeAnalysis.Location ReportingLocation => Location.symbol; + + bool IExpressionParentEntity.IsTopLevelParent => false; + + /// + /// Gets a string represention of a constant value. + /// + /// The value. + /// The string representation. + public static string ValueAsString(object value) + { + return value == null ? "null" : value is bool ? ((bool)value ? "true" : "false") : value.ToString(); + } + + /// + /// Creates an expression from a syntax node. + /// Inserts type conversion as required. + /// + /// The extraction context. + /// The node to extract. + /// The parent entity. + /// The child index. + /// A type hint. + /// The new expression. + public static Expression Create(Context cx, ExpressionSyntax node, IExpressionParentEntity parent, int child) => + CreateFromNode(new ExpressionNodeInfo(cx, node, parent, child)); + + public static Expression CreateFromNode(ExpressionNodeInfo info) => Expressions.ImplicitCast.Create(info); + + /// + /// Creates an expression from a syntax node. + /// Inserts type conversion as required. + /// Population is deferred to avoid overflowing the stack. + /// + /// The extraction context. + /// The node to extract. + /// The parent entity. + /// The child index. + /// A type hint. + public static void CreateDeferred(Context cx, ExpressionSyntax node, IExpressionParentEntity parent, int child) + { + if (ContainsPattern(node)) + // Expressions with patterns should be created right away, as they may introduce + // local variables referenced in `LocalVariable::GetAlreadyCreated()` + Create(cx, node, parent, child); + else + cx.PopulateLater(() => Create(cx, node, parent, child)); + } + + static bool ContainsPattern(SyntaxNode node) => + node is PatternSyntax || node is VariableDesignationSyntax || node.ChildNodes().Any(ContainsPattern); + + /// + /// Adapt the operator kind depending on whether it's a dynamic call or a user-operator call. + /// + /// + /// + /// + /// + public static ExprKind UnaryOperatorKind(Context cx, ExprKind originalKind, ExpressionSyntax node) => + GetCallType(cx, node).AdjustKind(originalKind); + + /// + /// If the expression calls an operator, add an expr_call() + /// to show the target of the call. Also note the dynamic method + /// name if available. + /// + /// Context + /// The expression. + public void OperatorCall(ExpressionSyntax node) + { + var @operator = cx.GetSymbolInfo(node); + var method = @operator.Symbol as IMethodSymbol; + + if (GetCallType(cx, node) == CallType.Dynamic) + { + UserOperator.OperatorSymbol(method.Name, out string operatorName); + cx.Emit(Tuples.dynamic_member_name(this, operatorName)); + return; + } + + if (method != null) + cx.Emit(Tuples.expr_call(this, Method.Create(cx, method))); + } + + public enum CallType + { + None, + BuiltInOperator, + Dynamic, + UserOperator + } + + /// + /// Determine what type of method was called for this expression. + /// + /// The context. + /// The expression + /// The call type. + public static CallType GetCallType(Context cx, ExpressionSyntax node) + { + var @operator = cx.GetSymbolInfo(node); + + if (@operator.Symbol != null) + { + var method = @operator.Symbol as IMethodSymbol; + + var containingSymbol = method.ContainingSymbol as ITypeSymbol; + if (containingSymbol != null && containingSymbol.TypeKind == Microsoft.CodeAnalysis.TypeKind.Dynamic) + { + return CallType.Dynamic; + } + + switch (method.MethodKind) + { + case MethodKind.BuiltinOperator: + if (method.ContainingType != null && method.ContainingType.TypeKind == Microsoft.CodeAnalysis.TypeKind.Delegate) + return CallType.UserOperator; + return CallType.BuiltInOperator; + default: + return CallType.UserOperator; + } + } + + return CallType.None; + } + + + public static bool IsDynamic(Context cx, ExpressionSyntax node) + { + var ti = cx.GetTypeInfo(node).ConvertedType; + return ti != null && ti.TypeKind == Microsoft.CodeAnalysis.TypeKind.Dynamic; + } + + /// + /// Given b in a?.b.c, return a. + /// + /// A MemberBindingExpression. + /// The qualifier of the conditional access. + protected static ExpressionSyntax FindConditionalQualifier(ExpressionSyntax node) + { + for (SyntaxNode n = node; n != null; n = n.Parent) + { + var conditionalAccess = n.Parent as ConditionalAccessExpressionSyntax; + + if (conditionalAccess != null && conditionalAccess.WhenNotNull == n) + return conditionalAccess.Expression; + } + + throw new InternalError(node, "Unable to locate a ConditionalAccessExpression"); + } + + public void MakeConditional() + { + cx.Emit(Tuples.conditional_access(this)); + } + + public void PopulateArguments(Context cx, BaseArgumentListSyntax args, int child) + { + foreach (var arg in args.Arguments) + PopulateArgument(cx, arg, child++); + } + + private void PopulateArgument(Context cx, ArgumentSyntax arg, int child) + { + var expr = Create(cx, arg.Expression, this, child); + int mode; + switch (arg.RefOrOutKeyword.Kind()) + { + case SyntaxKind.RefKeyword: + mode = 1; + break; + case SyntaxKind.OutKeyword: + mode = 2; + break; + case SyntaxKind.None: + mode = 0; + break; + default: + throw new InternalError(arg, "Unknown argument type"); + } + cx.Emit(Tuples.expr_argument(expr, mode)); + + if (arg.NameColon != null) + { + cx.Emit(Tuples.expr_argument_name(expr, arg.NameColon.Name.Identifier.Text)); + } + } + + public override string ToString() => Label.ToString(); + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.OptionalLabel; + } + + static class CallTypeExtensions + { + /// + /// Adjust the expression kind to match this call type. + /// + public static ExprKind AdjustKind(this Expression.CallType ct, ExprKind k) + { + switch (ct) + { + case Expression.CallType.Dynamic: + case Expression.CallType.UserOperator: + return ExprKind.OPERATOR_INVOCATION; + default: + return k; + } + } + } + + abstract class Expression : Expression + where SyntaxNode : ExpressionSyntax + { + public readonly SyntaxNode Syntax; + + protected Expression(ExpressionNodeInfo info) + : base(info) + { + Syntax = (SyntaxNode)info.Node; + } + + /// + /// Populates expression-type specific relations in the trap file. The general relations + /// expressions and expr_location are populated by the constructor + /// (should not fail), so even if expression-type specific population fails (e.g., in + /// standalone extraction), the expression created via + /// will + /// still be valid. + /// + protected abstract void Populate(); + + protected Expression TryPopulate() + { + cx.Try(Syntax, null, Populate); + return this; + } + } + + /// + /// Holds all information required to create an Expression entity. + /// + interface IExpressionInfo + { + Context Context { get; } + + /// + /// The type of the expression. + /// + Type Type { get; } + + /// + /// The location of the expression. + /// + Extraction.Entities.Location Location { get; } + + /// + /// The kind of the expression. + /// + ExprKind Kind { get; } + + /// + /// The parent of the expression. + /// + IExpressionParentEntity Parent { get; } + + /// + /// The child index of the expression. + /// + int Child { get; } + + /// + /// Holds if this is an implicit expression. + /// + bool IsCompilerGenerated { get; } + + /// + /// Gets a string representation of the value. + /// null is encoded as the string "null". + /// If the expression does not have a value, then this + /// is null. + /// + string ExprValue { get; } + } + + /// + /// Explicitly constructed expression information. + /// + class ExpressionInfo : IExpressionInfo + { + public Context Context { get; } + public Type Type { get; } + public Extraction.Entities.Location Location { get; } + public ExprKind Kind { get; } + public IExpressionParentEntity Parent { get; } + public int Child { get; } + public bool IsCompilerGenerated { get; } + public string ExprValue { get; } + + public ExpressionInfo(Context cx, Type type, Extraction.Entities.Location location, ExprKind kind, IExpressionParentEntity parent, int child, bool isCompilerGenerated, string value) + { + Context = cx; + Type = type; + Location = location; + Kind = kind; + Parent = parent; + Child = child; + ExprValue = value; + IsCompilerGenerated = isCompilerGenerated; + } + } + + /// + /// Expression information constructed from a syntax node. + /// + class ExpressionNodeInfo : IExpressionInfo + { + public ExpressionNodeInfo(Context cx, ExpressionSyntax node, IExpressionParentEntity parent, int child) : + this(cx, node, parent, child, cx.GetTypeInfo(node)) + { + } + + public ExpressionNodeInfo(Context cx, ExpressionSyntax node, IExpressionParentEntity parent, int child, TypeInfo typeInfo) + { + Context = cx; + Node = node; + Parent = parent; + Child = child; + TypeInfo = typeInfo; + Conversion = cx.Model(node).GetConversion(node); + } + + public ExpressionNodeInfo(Context cx, ExpressionSyntax node, IExpressionParentEntity parent, int child, ITypeSymbol type) : + this(cx, node, parent, child) + { + Type = Type.Create(cx, type); + } + + public Context Context { get; } + public ExpressionSyntax Node { get; private set; } + public IExpressionParentEntity Parent { get; set; } + public int Child { get; set; } + public TypeInfo TypeInfo { get; } + public Microsoft.CodeAnalysis.CSharp.Conversion Conversion { get; } + + public ITypeSymbol ResolvedType => Context.DisambiguateType(TypeInfo.Type); + public ITypeSymbol ConvertedType => Context.DisambiguateType(TypeInfo.ConvertedType); + + public ITypeSymbol ExpressionType + { + get + { + var type = ResolvedType; + + if (type == null) + type = Context.DisambiguateType(TypeInfo.Type ?? TypeInfo.ConvertedType); + + // Roslyn workaround: It can't work out the type of "new object[0]" + // Clearly a bug. + if (type != null && type.TypeKind == Microsoft.CodeAnalysis.TypeKind.Error) + { + var arrayCreation = Node as ArrayCreationExpressionSyntax; + if (arrayCreation != null) + { + var elementType = Context.GetType(arrayCreation.Type.ElementType); + + if (elementType != null) + return Context.Compilation.CreateArrayTypeSymbol(elementType, arrayCreation.Type.RankSpecifiers.Count); + } + + Context.ModelError(Node, "Failed to determine type"); + } + + return type; + } + } + + Microsoft.CodeAnalysis.Location location; + + public Microsoft.CodeAnalysis.Location CodeAnalysisLocation + { + get + { + if (location == null) + location = Node.FixedLocation(); + return location; + } + set + { + location = value; + } + } + + public SemanticModel Model => Context.Model(Node); + + public string ExprValue + { + get + { + var c = Model.GetConstantValue(Node); + return c.HasValue ? Expression.ValueAsString(c.Value) : null; + } + } + + Type cachedType; + + public Type Type + { + get + { + if (cachedType == null) + cachedType = Type.Create(Context, ExpressionType); + return cachedType; + } + set + { + cachedType = value; + } + } + + Extraction.Entities.Location cachedLocation; + + public Extraction.Entities.Location Location + { + get + { + if (cachedLocation == null) + cachedLocation = Context.Create(CodeAnalysisLocation); + return cachedLocation; + } + + set + { + cachedLocation = value; + } + } + + public ExprKind Kind { get; set; } = ExprKind.UNKNOWN; + + public bool IsCompilerGenerated { get; set; } + + public ExpressionNodeInfo SetParent(IExpressionParentEntity parent, int child) + { + Parent = parent; + Child = child; + return this; + } + + public ExpressionNodeInfo SetKind(ExprKind kind) + { + Kind = kind; + return this; + } + + public ExpressionNodeInfo SetType(Type type) + { + Type = type; + return this; + } + + public ExpressionNodeInfo SetNode(ExpressionSyntax node) + { + Node = node; + return this; + } + + SymbolInfo cachedSymbolInfo; + + public SymbolInfo SymbolInfo + { + get + { + if (cachedSymbolInfo.Symbol == null && cachedSymbolInfo.CandidateReason == CandidateReason.None) + cachedSymbolInfo = Model.GetSymbolInfo(Node); + return cachedSymbolInfo; + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Access.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Access.cs new file mode 100644 index 00000000000..60921d683c1 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Access.cs @@ -0,0 +1,55 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Access : Expression + { + static ExprKind AccessKind(Context cx, ISymbol symbol) + { + switch (symbol.Kind) + { + case SymbolKind.TypeParameter: + case SymbolKind.NamedType: + return ExprKind.TYPE_ACCESS; + + case SymbolKind.Field: + return ExprKind.FIELD_ACCESS; + + case SymbolKind.Property: + return ExprKind.PROPERTY_ACCESS; + + case SymbolKind.Event: + return ExprKind.EVENT_ACCESS; + + case SymbolKind.Method: + return ExprKind.METHOD_ACCESS; + + case SymbolKind.Local: + case SymbolKind.RangeVariable: + return ExprKind.LOCAL_VARIABLE_ACCESS; + + case SymbolKind.Parameter: + return ExprKind.PARAMETER_ACCESS; + + default: + cx.ModelError(symbol, "Unhandled access kind '{0}'", symbol.Kind); + return ExprKind.UNKNOWN; + } + } + + Access(ExpressionNodeInfo info, ISymbol symbol, bool implicitThis, IEntity target) + : base(info.SetKind(AccessKind(info.Context, symbol))) + { + cx.Emit(Tuples.expr_access(this, target)); + + if (implicitThis && !symbol.IsStatic) + { + This.CreateImplicit(cx, Type.Create(cx, symbol.ContainingType), Location, this, -1); + } + } + + public static Expression Create(ExpressionNodeInfo info, ISymbol symbol, bool implicitThis, IEntity target) => new Access(info, symbol, implicitThis, target); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ArgList.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ArgList.cs new file mode 100644 index 00000000000..7b8fc26f0a4 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ArgList.cs @@ -0,0 +1,18 @@ +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using System; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class ArgList : Expression + { + ArgList(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.UNKNOWN)) { } + + protected override void Populate() + { + throw new NotImplementedException(); + } + + public static ArgList Create(ExpressionNodeInfo info) => new ArgList(info); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ArrayCreation.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ArrayCreation.cs new file mode 100644 index 00000000000..c6f1a76a9d0 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ArrayCreation.cs @@ -0,0 +1,108 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + abstract class ArrayCreation : Expression where SyntaxNode : ExpressionSyntax + { + protected ArrayCreation(ExpressionNodeInfo info) : base(info) { } + } + + class StackAllocArrayCreation : ArrayCreation + { + StackAllocArrayCreation(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.ARRAY_CREATION)) { } + + public static Expression Create(ExpressionNodeInfo info) => new StackAllocArrayCreation(info).TryPopulate(); + + protected override void Populate() + { + var arrayType = Syntax.Type as ArrayTypeSyntax; + + if (arrayType == null) + { + cx.ModelError(Syntax, "Unexpected array type"); + return; + } + + var child = 0; + + foreach (var rank in arrayType.RankSpecifiers.SelectMany(rs => rs.Sizes)) + { + Create(cx, rank, this, child++); + } + + cx.Emit(Tuples.explicitly_sized_array_creation(this)); + } + } + + class ExplicitArrayCreation : ArrayCreation + { + ExplicitArrayCreation(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.ARRAY_CREATION)) { } + + public static Expression Create(ExpressionNodeInfo info) => new ExplicitArrayCreation(info).TryPopulate(); + + protected override void Populate() + { + var child = 0; + bool explicitlySized = false; + + foreach (var rank in Syntax.Type.RankSpecifiers.SelectMany(rs => rs.Sizes)) + { + if (rank is OmittedArraySizeExpressionSyntax) + { + // Create an expression which simulates the explicit size of the array + + if (Syntax.Initializer != null) + { + // An implicitly-sized array must have an initializer. + // Guard it just in case. + var size = Syntax.Initializer.Expressions.Count; + + var info = new ExpressionInfo( + cx, + Type.Create(cx, cx.Compilation.GetSpecialType(Microsoft.CodeAnalysis.SpecialType.System_Int32)), + Location, + ExprKind.INT_LITERAL, + this, + child, + false, + size.ToString()); + + new Expression(info); + } + } + else + { + Create(cx, rank, this, child); + explicitlySized = true; + } + child++; + } + if (Syntax.Initializer != null) + { + ArrayInitializer.Create(new ExpressionNodeInfo(cx, Syntax.Initializer, this, -1)); + } + + if (explicitlySized) + cx.Emit(Tuples.explicitly_sized_array_creation(this)); + } + } + + class ImplicitArrayCreation : ArrayCreation + { + ImplicitArrayCreation(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.ARRAY_CREATION)) { } + + public static Expression Create(ExpressionNodeInfo info) => new ImplicitArrayCreation(info).TryPopulate(); + + protected override void Populate() + { + if (Syntax.Initializer != null) + { + ArrayInitializer.Create(new ExpressionNodeInfo(cx, Syntax.Initializer, this, -1)); + } + + cx.Emit(Tuples.implicitly_typed_array_creation(this)); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Assignment.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Assignment.cs new file mode 100644 index 00000000000..921787cabc5 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Assignment.cs @@ -0,0 +1,154 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Microsoft.CodeAnalysis.CSharp; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Assignment : Expression + { + Assignment(ExpressionNodeInfo info) + : base(info.SetKind(GetKind(info.Context, (AssignmentExpressionSyntax)info.Node))) + { + } + + public static Assignment Create(ExpressionNodeInfo info) + { + var ret = new Assignment(info); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + var operatorKind = OperatorKind; + if (operatorKind.HasValue) + { + // Convert assignment such as `a += b` into `a = a + b`. + var simpleAssignExpr = new Expression(new ExpressionInfo(cx, Type, Location, ExprKind.SIMPLE_ASSIGN, this, 2, false, null)); + Create(cx, Syntax.Left, simpleAssignExpr, 1); + var opexpr = new Expression(new ExpressionInfo(cx, Type, Location, operatorKind.Value, simpleAssignExpr, 0, false, null)); + Create(cx, Syntax.Left, opexpr, 0); + Create(cx, Syntax.Right, opexpr, 1); + opexpr.OperatorCall(Syntax); + } + else + { + Create(cx, Syntax.Left, this, 1); + Create(cx, Syntax.Right, this, 0); + + if (Kind == ExprKind.ADD_EVENT || Kind == ExprKind.REMOVE_EVENT) + { + OperatorCall(Syntax); + } + } + } + + static ExprKind GetAssignmentOperation(Context cx, AssignmentExpressionSyntax syntax) + { + switch (syntax.OperatorToken.Kind()) + { + case SyntaxKind.PlusEqualsToken: + return ExprKind.ASSIGN_ADD; + case SyntaxKind.MinusEqualsToken: + return ExprKind.ASSIGN_SUB; + case SyntaxKind.EqualsToken: + return ExprKind.SIMPLE_ASSIGN; + case SyntaxKind.BarEqualsToken: + return ExprKind.ASSIGN_OR; + case SyntaxKind.AmpersandEqualsToken: + return ExprKind.ASSIGN_AND; + case SyntaxKind.CaretEqualsToken: + return ExprKind.ASSIGN_XOR; + case SyntaxKind.AsteriskEqualsToken: + return ExprKind.ASSIGN_MUL; + case SyntaxKind.PercentEqualsToken: + return ExprKind.ASSIGN_REM; + case SyntaxKind.SlashEqualsToken: + return ExprKind.ASSIGN_DIV; + case SyntaxKind.LessThanLessThanEqualsToken: + return ExprKind.ASSIGN_LSHIFT; + case SyntaxKind.GreaterThanGreaterThanEqualsToken: + return ExprKind.ASSIGN_RSHIFT; + default: + cx.ModelError(syntax, "Unrecognised assignment type " + GetKind(cx, syntax)); + return ExprKind.UNKNOWN; + } + } + + static ExprKind GetKind(Context cx, AssignmentExpressionSyntax syntax) + { + var leftSymbol = cx.GetSymbolInfo(syntax.Left); + bool assignEvent = leftSymbol.Symbol != null && leftSymbol.Symbol is IEventSymbol; + var kind = GetAssignmentOperation(cx, syntax); + var leftType = cx.GetType(syntax.Left); + + if (leftType != null && leftType.SpecialType != SpecialType.None) + { + // In Mono, the builtin types did not specify their operator invocation + // even though EVERY operator has an invocation in C#. (This is a flaw in the dbscheme and should be fixed). + return kind; + } + + if (kind == ExprKind.ASSIGN_ADD && assignEvent) + { + return ExprKind.ADD_EVENT; + } + + if (kind == ExprKind.ASSIGN_SUB && assignEvent) + { + return ExprKind.REMOVE_EVENT; + } + + return kind; + } + + /// + /// Gets the kind of this assignment operator (null if the + /// assignment is not an assignment operator). For example, the operator + /// kind of `*=` is `*`. + /// + ExprKind? OperatorKind + { + get + { + var kind = Kind; + if (kind == ExprKind.REMOVE_EVENT || kind == ExprKind.ADD_EVENT || kind == ExprKind.SIMPLE_ASSIGN) + return null; + + if (CallType.AdjustKind(kind) == ExprKind.OPERATOR_INVOCATION) + return ExprKind.OPERATOR_INVOCATION; + + switch (kind) + { + case ExprKind.ASSIGN_ADD: + return ExprKind.ADD; + case ExprKind.ASSIGN_AND: + return ExprKind.BIT_AND; + case ExprKind.ASSIGN_DIV: + return ExprKind.DIV; + case ExprKind.ASSIGN_LSHIFT: + return ExprKind.LSHIFT; + case ExprKind.ASSIGN_MUL: + return ExprKind.MUL; + case ExprKind.ASSIGN_OR: + return ExprKind.BIT_OR; + case ExprKind.ASSIGN_REM: + return ExprKind.REM; + case ExprKind.ASSIGN_RSHIFT: + return ExprKind.RSHIFT; + case ExprKind.ASSIGN_SUB: + return ExprKind.SUB; + case ExprKind.ASSIGN_XOR: + return ExprKind.BIT_XOR; + default: + cx.ModelError(Syntax, "Couldn't unfold assignment of type " + kind); + return ExprKind.UNKNOWN; + } + } + } + + public new CallType CallType => GetCallType(cx, Syntax); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Await.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Await.cs new file mode 100644 index 00000000000..d7c758a3539 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Await.cs @@ -0,0 +1,17 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Await : Expression + { + Await(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.AWAIT)) { } + + public static Expression Create(ExpressionNodeInfo info) => new Await(info).TryPopulate(); + + protected override void Populate() + { + Create(cx, Syntax.Expression, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Base.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Base.cs new file mode 100644 index 00000000000..e67c18245f6 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Base.cs @@ -0,0 +1,12 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Base : Expression + { + Base(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.BASE_ACCESS)) { } + + public static Base Create(ExpressionNodeInfo info) => new Base(info); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Binary.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Binary.cs new file mode 100644 index 00000000000..afe1f351b21 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Binary.cs @@ -0,0 +1,61 @@ +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Binary : Expression + { + Binary(ExpressionNodeInfo info) + : base(info.SetKind(GetKind(info.Context, (BinaryExpressionSyntax)info.Node))) + { + } + + public static Expression Create(ExpressionNodeInfo info) => new Binary(info).TryPopulate(); + + protected override void Populate() + { + OperatorCall(Syntax); + CreateDeferred(cx, Syntax.Left, this, 0); + CreateDeferred(cx, Syntax.Right, this, 1); + } + + static ExprKind GetKind(Context cx, BinaryExpressionSyntax node) + { + var k = GetBinaryTokenKind(cx, node.OperatorToken.Kind()); + return GetCallType(cx, node).AdjustKind(k); + } + + static ExprKind GetBinaryTokenKind(Context cx, SyntaxKind kind) + { + switch (kind) + { + case SyntaxKind.LessThanToken: return ExprKind.LT; + case SyntaxKind.PlusToken: return ExprKind.ADD; + case SyntaxKind.LessThanEqualsToken: return ExprKind.LE; + case SyntaxKind.GreaterThanToken: return ExprKind.GT; + case SyntaxKind.AsteriskToken: return ExprKind.MUL; + case SyntaxKind.AmpersandAmpersandToken: return ExprKind.LOG_AND; + case SyntaxKind.EqualsEqualsToken: return ExprKind.EQ; + case SyntaxKind.PercentToken: return ExprKind.REM; + case SyntaxKind.MinusToken: return ExprKind.SUB; + case SyntaxKind.AmpersandToken: return ExprKind.BIT_AND; + case SyntaxKind.BarToken: return ExprKind.BIT_OR; + case SyntaxKind.SlashToken: return ExprKind.DIV; + case SyntaxKind.ExclamationEqualsToken: return ExprKind.NE; + case SyntaxKind.AsKeyword: return ExprKind.AS; + case SyntaxKind.IsKeyword: return ExprKind.IS; + case SyntaxKind.BarBarToken: return ExprKind.LOG_OR; + case SyntaxKind.GreaterThanEqualsToken: return ExprKind.GE; + case SyntaxKind.GreaterThanGreaterThanToken: return ExprKind.RSHIFT; + case SyntaxKind.LessThanLessThanToken: return ExprKind.LSHIFT; + case SyntaxKind.CaretToken: return ExprKind.BIT_XOR; + case SyntaxKind.QuestionQuestionToken: return ExprKind.NULL_COALESCING; + // !! And the rest + default: + cx.ModelError("Unhandled operator type {0}", kind); + return ExprKind.UNKNOWN; + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Cast.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Cast.cs new file mode 100644 index 00000000000..8a99f15f5bf --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Cast.cs @@ -0,0 +1,29 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Cast : Expression + { + Cast(ExpressionNodeInfo info) : base(info.SetKind(UnaryOperatorKind(info.Context, ExprKind.CAST, info.Node))) { } + + public static Expression Create(ExpressionNodeInfo info) => new Cast(info).TryPopulate(); + + protected override void Populate() + { + Create(cx, Syntax.Expression, this, 0); + + if (Kind == ExprKind.CAST) + // Type cast + TypeAccess.Create(new ExpressionNodeInfo(cx, Syntax.Type, this, 1)); + else + { + // Type conversion + OperatorCall(Syntax); + TypeMention.Create(cx, Syntax.Type, this, Type); + } + } + + public override Microsoft.CodeAnalysis.Location ReportingLocation => Syntax.GetLocation(); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Checked.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Checked.cs new file mode 100644 index 00000000000..53c8f103dfa --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Checked.cs @@ -0,0 +1,17 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Checked : Expression + { + Checked(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.CHECKED)) { } + + public static Expression Create(ExpressionNodeInfo info) => new Checked(info).TryPopulate(); + + protected override void Populate() + { + Create(cx, Syntax.Expression, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Conditional.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Conditional.cs new file mode 100644 index 00000000000..5206e555ce1 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Conditional.cs @@ -0,0 +1,19 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Conditional : Expression + { + Conditional(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.CONDITIONAL)) { } + + public static Expression Create(ExpressionNodeInfo info) => new Conditional(info).TryPopulate(); + + protected override void Populate() + { + Create(cx, Syntax.Condition, this, 0); + Create(cx, Syntax.WhenTrue, this, 1); + Create(cx, Syntax.WhenFalse, this, 2); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Default.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Default.cs new file mode 100644 index 00000000000..b0470496b3e --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Default.cs @@ -0,0 +1,17 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Default : Expression + { + Default(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.DEFAULT)) { } + + public static Expression Create(ExpressionNodeInfo info) => new Default(info).TryPopulate(); + + protected override void Populate() + { + TypeAccess.Create(cx, Syntax.Type, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Discard.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Discard.cs new file mode 100644 index 00000000000..32a8f373ef8 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Discard.cs @@ -0,0 +1,16 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Discard : Expression + { + public Discard(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.DISCARD)) + { + } + + protected override void Populate() + { + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ElementAccess.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ElementAccess.cs new file mode 100644 index 00000000000..b979f8ca3a9 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ElementAccess.cs @@ -0,0 +1,95 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + abstract class ElementAccess : Expression + { + protected ElementAccess(ExpressionNodeInfo info, ExpressionSyntax qualifier, BracketedArgumentListSyntax argumentList) + : base(info.SetKind(GetKind(info.Context, qualifier))) + { + Qualifier = qualifier; + ArgumentList = argumentList; + } + + readonly ExpressionSyntax Qualifier; + readonly BracketedArgumentListSyntax ArgumentList; + + protected override void Populate() + { + if (Kind == ExprKind.POINTER_INDIRECTION) + { + var qualifierInfo = new ExpressionNodeInfo(cx, Qualifier, this, 0); + var add = new Expression(new ExpressionInfo(cx, qualifierInfo.Type, Location, ExprKind.ADD, this, 0, false, null)); + qualifierInfo.SetParent(add, 0); + CreateFromNode(qualifierInfo); + PopulateArguments(cx, ArgumentList, 1); + } + else + { + var child = -1; + Create(cx, Qualifier, this, child++); + foreach (var a in ArgumentList.Arguments) + { + cx.Extract(a, this, child++); + } + + var symbolInfo = cx.GetSymbolInfo(base.Syntax); + + var indexer = symbolInfo.Symbol as IPropertySymbol; + if (indexer != null) + { + cx.Emit(Tuples.expr_access(this, Indexer.Create(cx, indexer))); + } + } + } + + public sealed override Microsoft.CodeAnalysis.Location ReportingLocation => base.ReportingLocation; + + static ExprKind GetKind(Context cx, ExpressionSyntax qualifier) + { + var qualifierType = cx.GetType(qualifier); + + // This is a compilation error, so make a guess and continue. + if (qualifierType == null) return ExprKind.ARRAY_ACCESS; + + if (qualifierType.TypeKind == Microsoft.CodeAnalysis.TypeKind.Pointer) + { + // Convert expressions of the form a[b] into *(a+b) + return ExprKind.POINTER_INDIRECTION; + } + + return IsDynamic(cx, qualifier) ? + ExprKind.DYNAMIC_ELEMENT_ACCESS : + qualifierType.TypeKind == Microsoft.CodeAnalysis.TypeKind.Array ? + ExprKind.ARRAY_ACCESS : + ExprKind.INDEXER_ACCESS; + } + } + + class NormalElementAccess : ElementAccess + { + NormalElementAccess(ExpressionNodeInfo info) + : base(info, ((ElementAccessExpressionSyntax)info.Node).Expression, ((ElementAccessExpressionSyntax)info.Node).ArgumentList) { } + + public static Expression Create(ExpressionNodeInfo info) => new NormalElementAccess(info).TryPopulate(); + } + + class BindingElementAccess : ElementAccess + { + BindingElementAccess(ExpressionNodeInfo info) + : base(info, FindConditionalQualifier(info.Node), ((ElementBindingExpressionSyntax)info.Node).ArgumentList) { } + + public static Expression Create(ExpressionNodeInfo info) => new BindingElementAccess(info).TryPopulate(); + + protected override void Populate() + { + base.Populate(); + MakeConditional(); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Factory.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Factory.cs new file mode 100644 index 00000000000..86e7164d4d8 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Factory.cs @@ -0,0 +1,242 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + internal static class Factory + { + internal static Expression Create(ExpressionNodeInfo info) + { + // Some expressions can be extremely deep (e.g. string + string + string ...) + // to the extent that the stack has been known to overflow. + using (info.Context.StackGuard) + { + if (info.Node == null) + { + info.Context.ModelError("Attempt to create a null expression"); + return new Unknown(info); + } + + switch (info.Node.Kind()) + { + case SyntaxKind.AddExpression: + case SyntaxKind.SubtractExpression: + case SyntaxKind.LessThanExpression: + case SyntaxKind.LessThanOrEqualExpression: + case SyntaxKind.GreaterThanExpression: + case SyntaxKind.GreaterThanOrEqualExpression: + case SyntaxKind.MultiplyExpression: + case SyntaxKind.LogicalAndExpression: + case SyntaxKind.EqualsExpression: + case SyntaxKind.ModuloExpression: + case SyntaxKind.BitwiseAndExpression: + case SyntaxKind.BitwiseOrExpression: + case SyntaxKind.DivideExpression: + case SyntaxKind.NotEqualsExpression: + case SyntaxKind.LogicalOrExpression: + case SyntaxKind.IsExpression: + case SyntaxKind.AsExpression: + case SyntaxKind.RightShiftExpression: + case SyntaxKind.LeftShiftExpression: + case SyntaxKind.ExclusiveOrExpression: + case SyntaxKind.CoalesceExpression: + return Binary.Create(info); + + case SyntaxKind.FalseLiteralExpression: + case SyntaxKind.TrueLiteralExpression: + case SyntaxKind.StringLiteralExpression: + case SyntaxKind.NullLiteralExpression: + case SyntaxKind.NumericLiteralExpression: + case SyntaxKind.CharacterLiteralExpression: + case SyntaxKind.DefaultLiteralExpression: + return Literal.Create(info); + + case SyntaxKind.InvocationExpression: + return Invocation.Create(info); + + case SyntaxKind.PostIncrementExpression: + return PostfixUnary.Create(info.SetKind(ExprKind.POST_INCR), ((PostfixUnaryExpressionSyntax)info.Node).Operand); + + case SyntaxKind.PostDecrementExpression: + return PostfixUnary.Create(info.SetKind(ExprKind.POST_DECR), ((PostfixUnaryExpressionSyntax)info.Node).Operand); + + case SyntaxKind.AwaitExpression: + return Await.Create(info); + + case SyntaxKind.ElementAccessExpression: + return NormalElementAccess.Create(info); + + case SyntaxKind.SimpleAssignmentExpression: + case SyntaxKind.OrAssignmentExpression: + case SyntaxKind.AndAssignmentExpression: + case SyntaxKind.SubtractAssignmentExpression: + case SyntaxKind.AddAssignmentExpression: + case SyntaxKind.MultiplyAssignmentExpression: + case SyntaxKind.ExclusiveOrAssignmentExpression: + case SyntaxKind.LeftShiftAssignmentExpression: + case SyntaxKind.RightShiftAssignmentExpression: + case SyntaxKind.DivideAssignmentExpression: + case SyntaxKind.ModuloAssignmentExpression: + return Assignment.Create(info); + + case SyntaxKind.ObjectCreationExpression: + return ExplicitObjectCreation.Create(info); + + case SyntaxKind.ArrayCreationExpression: + return ExplicitArrayCreation.Create(info); + + case SyntaxKind.ObjectInitializerExpression: + return ObjectInitializer.Create(info); + + case SyntaxKind.ArrayInitializerExpression: + return ImplicitArrayInitializer.Create(info); + + case SyntaxKind.CollectionInitializerExpression: + return CollectionInitializer.Create(info); + + case SyntaxKind.ConditionalAccessExpression: + return MemberAccess.Create(info, (ConditionalAccessExpressionSyntax)info.Node); + + case SyntaxKind.SimpleMemberAccessExpression: + return MemberAccess.Create(info, (MemberAccessExpressionSyntax)info.Node); + + case SyntaxKind.UnaryMinusExpression: + return Unary.Create(info.SetKind(ExprKind.MINUS)); + + case SyntaxKind.UnaryPlusExpression: + return Unary.Create(info.SetKind(ExprKind.PLUS)); + + case SyntaxKind.SimpleLambdaExpression: + return Lambda.Create(info, (SimpleLambdaExpressionSyntax)info.Node); + + case SyntaxKind.ParenthesizedLambdaExpression: + return Lambda.Create(info, (ParenthesizedLambdaExpressionSyntax)info.Node); + + case SyntaxKind.ConditionalExpression: + return Conditional.Create(info); + + case SyntaxKind.CastExpression: + return Cast.Create(info); + + case SyntaxKind.ParenthesizedExpression: + return Create(info.SetNode(((ParenthesizedExpressionSyntax)info.Node).Expression)); + + case SyntaxKind.PointerType: + case SyntaxKind.ArrayType: + case SyntaxKind.PredefinedType: + case SyntaxKind.NullableType: + return TypeAccess.Create(info); + + case SyntaxKind.TypeOfExpression: + return TypeOf.Create(info); + + case SyntaxKind.QualifiedName: + case SyntaxKind.IdentifierName: + case SyntaxKind.AliasQualifiedName: + case SyntaxKind.GenericName: + return Name.Create(info); + + case SyntaxKind.LogicalNotExpression: + return Unary.Create(info.SetKind(ExprKind.LOG_NOT)); + + case SyntaxKind.BitwiseNotExpression: + return Unary.Create(info.SetKind(ExprKind.BIT_NOT)); + + case SyntaxKind.PreIncrementExpression: + return Unary.Create(info.SetKind(ExprKind.PRE_INCR)); + + case SyntaxKind.PreDecrementExpression: + return Unary.Create(info.SetKind(ExprKind.PRE_DECR)); + + case SyntaxKind.ThisExpression: + return This.CreateExplicit(info); + + case SyntaxKind.AddressOfExpression: + return Unary.Create(info.SetKind(ExprKind.ADDRESS_OF)); + + case SyntaxKind.PointerIndirectionExpression: + return Unary.Create(info.SetKind(ExprKind.POINTER_INDIRECTION)); + + case SyntaxKind.DefaultExpression: + return Default.Create(info); + + case SyntaxKind.CheckedExpression: + return Checked.Create(info); + + case SyntaxKind.UncheckedExpression: + return Unchecked.Create(info); + + case SyntaxKind.BaseExpression: + return Base.Create(info); + + case SyntaxKind.AnonymousMethodExpression: + return Lambda.Create(info, (AnonymousMethodExpressionSyntax)info.Node); + + case SyntaxKind.ImplicitArrayCreationExpression: + return ImplicitArrayCreation.Create(info); + + case SyntaxKind.AnonymousObjectCreationExpression: + return ImplicitObjectCreation.Create(info); + + case SyntaxKind.ComplexElementInitializerExpression: + return CollectionInitializer.Create(info); + + case SyntaxKind.SizeOfExpression: + return SizeOf.Create(info); + + case SyntaxKind.PointerMemberAccessExpression: + return PointerMemberAccess.Create(info); + + case SyntaxKind.QueryExpression: + return Query.Create(info); + + case SyntaxKind.InterpolatedStringExpression: + return InterpolatedString.Create(info); + + case SyntaxKind.MemberBindingExpression: + return MemberAccess.Create(info, (MemberBindingExpressionSyntax)info.Node); + + case SyntaxKind.ElementBindingExpression: + return BindingElementAccess.Create(info); + + case SyntaxKind.StackAllocArrayCreationExpression: + return StackAllocArrayCreation.Create(info); + + case SyntaxKind.ArgListExpression: + return ArgList.Create(info); + + case SyntaxKind.RefTypeExpression: + return RefType.Create(info); + + case SyntaxKind.RefValueExpression: + return RefValue.Create(info); + + case SyntaxKind.MakeRefExpression: + return MakeRef.Create(info); + + case SyntaxKind.ThrowExpression: + return Throw.Create(info); + + case SyntaxKind.DeclarationExpression: + return VariableDeclaration.Create(info.Context, (DeclarationExpressionSyntax)info.Node, info.Parent, info.Child); + + case SyntaxKind.TupleExpression: + return Tuple.Create(info); + + case SyntaxKind.RefExpression: + return Ref.Create(info); + + case SyntaxKind.IsPatternExpression: + return IsPattern.Create(info); + + default: + info.Context.ModelError(info.Node, "Unhandled expression '{0}' of kind '{1}'", info.Node, info.Node.Kind()); + return new Unknown(info); + } + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ImplicitCast.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ImplicitCast.cs new file mode 100644 index 00000000000..f92102151ba --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ImplicitCast.cs @@ -0,0 +1,92 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class ImplicitCast : Expression + { + public Expression Expr + { + get; + private set; + } + + public ImplicitCast(ExpressionNodeInfo info) + : base(new ExpressionInfo(info.Context, Type.Create(info.Context, info.ConvertedType), info.Location, ExprKind.CAST, info.Parent, info.Child, true, info.ExprValue)) + { + Expr = Factory.Create(new ExpressionNodeInfo(cx, info.Node, this, 0)); + } + + public ImplicitCast(ExpressionNodeInfo info, IMethodSymbol method) + : base(new ExpressionInfo(info.Context, Type.Create(info.Context, info.ConvertedType), info.Location, ExprKind.OPERATOR_INVOCATION, info.Parent, info.Child, true, info.ExprValue) ) + { + Expr = Factory.Create(info.SetParent(this, 0)); + + var target = Method.Create(cx, method); + if (target != null) + cx.Emit(Tuples.expr_call(this, target)); + else + cx.ModelError(info.Node, "Failed to resolve target for operator invocation"); + } + + /// + /// Creates a new expression, adding casts as required. + /// + /// The extraction context. + /// The expression node. + /// The parent of the expression. + /// The child number. + /// A type hint. + /// A new expression. + public static Expression Create(ExpressionNodeInfo info) + { + var resolvedType = info.ResolvedType; + var convertedType = info.ConvertedType; + var conversion = info.Conversion; + + if (conversion.MethodSymbol != null) + { + bool convertedToDelegate = Type.IsDelegate(convertedType); + + if (convertedToDelegate) + { + var objectCreation = info.Parent as ExplicitObjectCreation; + bool isExplicitConversion = objectCreation != null && objectCreation.Kind == ExprKind.EXPLICIT_DELEGATE_CREATION; + + if (!isExplicitConversion) + { + info.Kind = ExprKind.IMPLICIT_DELEGATE_CREATION; + var parent = new Expression(info); + return Factory.Create(new ExpressionNodeInfo(info.Context, info.Node, parent, 0)); + } + + info.Kind = ExprKind.UNKNOWN; + return Factory.Create(info); + } + + if (resolvedType != null) + return new ImplicitCast(info, conversion.MethodSymbol); + } + + bool implicitUpcast = conversion.IsImplicit && + convertedType != null && + !conversion.IsBoxing && + ( + resolvedType == null || + conversion.IsReference || + convertedType.SpecialType == SpecialType.System_Object) + ; + + if (!conversion.IsIdentity && !implicitUpcast) + { + return new ImplicitCast(info); + } + + // Default: Just create the expression without a conversion. + return Factory.Create(info); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Initializer.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Initializer.cs new file mode 100644 index 00000000000..317ce29310e --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Initializer.cs @@ -0,0 +1,137 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Entities; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + abstract class Initializer : Expression + { + protected Initializer(ExpressionNodeInfo info) : base(info) { } + } + + class ArrayInitializer : Expression + { + ArrayInitializer(ExpressionNodeInfo info) : base(info.SetType(Type.Create(info.Context, null)).SetKind(ExprKind.ARRAY_INIT)) { } + + public static Expression Create(ExpressionNodeInfo info) => new ArrayInitializer(info).TryPopulate(); + + protected override void Populate() + { + var child = 0; + foreach (var e in Syntax.Expressions) + { + if (e.Kind() == SyntaxKind.ArrayInitializerExpression) + { + // Recursively create another array initializer + Create(new ExpressionNodeInfo(cx, (InitializerExpressionSyntax)e, this, child++)); + } + else + { + // Create the expression normally. + Create(cx, e, this, child++); + } + } + } + } + + // Array initializer { ..., ... }. + class ImplicitArrayInitializer : Initializer + { + ImplicitArrayInitializer(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.ARRAY_CREATION)) { } + + public static Expression Create(ExpressionNodeInfo info) => new ImplicitArrayInitializer(info).TryPopulate(); + + protected override void Populate() + { + ArrayInitializer.Create(new ExpressionNodeInfo(cx, Syntax, this, -1)); + cx.Emit(Tuples.implicitly_typed_array_creation(this)); + } + } + + class ObjectInitializer : Initializer + { + ObjectInitializer(ExpressionNodeInfo info) + : base(info.SetKind(ExprKind.OBJECT_INIT)) { } + + public static Expression Create(ExpressionNodeInfo info) => new ObjectInitializer(info).TryPopulate(); + + protected override void Populate() + { + var child = 0; + + foreach (var init in Syntax.Expressions) + { + var assignment = init as AssignmentExpressionSyntax; + + if (assignment != null) + { + var assignmentEntity = new Expression(new ExpressionNodeInfo(cx, init, this, child++).SetKind(ExprKind.SIMPLE_ASSIGN)); + + CreateFromNode(new ExpressionNodeInfo(cx, assignment.Right, assignmentEntity, 0)); + + var target = cx.GetSymbolInfo(assignment.Left); + if (target.Symbol == null) + { + cx.ModelError(assignment, "Unknown object initializer"); + new Unknown(new ExpressionNodeInfo(cx, assignment.Left, assignmentEntity, 1)); + } + else + { + Access.Create(new ExpressionNodeInfo(cx, assignment.Left, assignmentEntity, 1), target.Symbol, false, cx.CreateEntity(target.Symbol)); + } + } + else + { + cx.ModelError(init, "Unexpected object initialization"); + Create(cx, init, this, child++); + } + } + } + } + + class CollectionInitializer : Initializer + { + CollectionInitializer(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.COLLECTION_INIT)) { } + + public static Expression Create(ExpressionNodeInfo info) => new CollectionInitializer(info).TryPopulate(); + + protected override void Populate() + { + var child = 0; + foreach (var i in Syntax.Expressions) + { + var collectionInfo = cx.Model(Syntax).GetCollectionInitializerSymbolInfo(i); + var addMethod = Method.Create(cx, collectionInfo.Symbol as IMethodSymbol); + var voidType = Type.Create(cx, cx.Compilation.GetSpecialType(SpecialType.System_Void)); + + var invocation = new Expression(new ExpressionInfo(cx, voidType, cx.Create(i.GetLocation()), ExprKind.METHOD_INVOCATION, this, child++, false, null)); + + if (addMethod != null) + cx.Emit(Tuples.expr_call(invocation, addMethod)); + else + cx.ModelError(Syntax, "Unable to find an Add() method for collection initializer"); + + if (i.Kind() == SyntaxKind.ComplexElementInitializerExpression) + { + // Arrays of the form new Foo { { 1,2 }, { 3, 4 } } + // where the arguments { 1, 2 } are passed to the Add() method. + + var init = (InitializerExpressionSyntax)i; + + int addChild = 0; + foreach (var arg in init.Expressions) + { + Create(cx, arg, invocation, addChild++); + } + } + else + { + Create(cx, i, invocation, 0); + } + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/InterpolatedString.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/InterpolatedString.cs new file mode 100644 index 00000000000..fa24766bb8f --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/InterpolatedString.cs @@ -0,0 +1,36 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis.CSharp; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class InterpolatedString : Expression + { + InterpolatedString(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.INTERPOLATED_STRING)) { } + + public static Expression Create(ExpressionNodeInfo info) => new InterpolatedString(info).TryPopulate(); + + protected override void Populate() + { + var child = 0; + foreach (var c in Syntax.Contents) + { + switch (c.Kind()) + { + case SyntaxKind.Interpolation: + var interpolation = (InterpolationSyntax)c; + Create(cx, interpolation.Expression, this, child++); + break; + case SyntaxKind.InterpolatedStringText: + // Create a string literal + var interpolatedText = (InterpolatedStringTextSyntax)c; + new Expression(new ExpressionInfo(cx, Type, cx.Create(c.GetLocation()), ExprKind.STRING_LITERAL, this, child++, false, interpolatedText.TextToken.Text)); + break; + default: + throw new InternalError(c, "Unhandled interpolation kind {0}", c.Kind()); + } + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Invocation.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Invocation.cs new file mode 100644 index 00000000000..750034dc771 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Invocation.cs @@ -0,0 +1,181 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using System.Linq; +using Microsoft.CodeAnalysis.CSharp; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Invocation : Expression + { + Invocation(ExpressionNodeInfo info) + : base(info.SetKind(GetKind(info))) + { + this.info = info; + } + + readonly ExpressionNodeInfo info; + + public static Expression Create(ExpressionNodeInfo info) => new Invocation(info).TryPopulate(); + + protected override void Populate() + { + if (IsNameof(Syntax)) + { + PopulateArguments(cx, Syntax.ArgumentList, 0); + return; + } + + var child = -1; + string memberName = null; + var target = TargetSymbol; + switch (Syntax.Expression) + { + case MemberAccessExpressionSyntax memberAccess: + memberName = memberAccess.Name.Identifier.Text; + if (Syntax.Expression.Kind() == SyntaxKind.SimpleMemberAccessExpression) + // Qualified method call; `x.M()` + Create(cx, memberAccess.Expression, this, child++); + else + // Pointer member access; `x->M()` + Create(cx, Syntax.Expression, this, child++); + break; + case MemberBindingExpressionSyntax memberBinding: + // Conditionally qualified method call; `x?.M()` + memberName = memberBinding.Name.Identifier.Text; + Create(cx, FindConditionalQualifier(memberBinding), this, child++); + MakeConditional(); + break; + case SimpleNameSyntax simpleName when (Kind == ExprKind.METHOD_INVOCATION): + // Unqualified method call; `M()` + memberName = simpleName.Identifier.Text; + if (target != null && !target.IsStatic) + { + // Implicit `this` qualifier; add explicitly + var callingMethod = cx.Model(Syntax).GetEnclosingSymbol(Location.symbol.SourceSpan.Start) as IMethodSymbol; + + if (callingMethod == null) + cx.ModelError(Syntax, "Couldn't determine implicit this type"); + else + This.CreateImplicit(cx, Type.Create(cx, callingMethod.ContainingType), Location, this, child++); + } + else + { + // No implicit `this` qualifier + child++; + } + break; + default: + // Delegate call; `d()` + Create(cx, Syntax.Expression, this, child++); + break; + } + + var isDynamicCall = IsDynamicCall(info); + if (isDynamicCall) + { + if (memberName != null) + cx.Emit(Tuples.dynamic_member_name(this, memberName)); + else + cx.ModelError(Syntax, "Unable to get name for dynamic call."); + } + + PopulateArguments(cx, Syntax.ArgumentList, child); + + if (target == null) + { + if (!isDynamicCall && !IsDelegateCall(info)) + cx.ModelError(Syntax, "Unable to resolve target for call. (Compilation error?)"); + return; + } + + var targetKey = Method.Create(cx, target); + cx.Emit(Tuples.expr_call(this, targetKey)); + } + + static bool IsDynamicCall(ExpressionNodeInfo info) + { + // Either the qualifier (Expression) is dynamic, + // or one of the arguments is dynamic. + var node = (InvocationExpressionSyntax)info.Node; + return !IsDelegateCall(info) && + (IsDynamic(info.Context, node.Expression) || node.ArgumentList.Arguments.Any(arg => IsDynamic(info.Context, arg.Expression))); + } + + public SymbolInfo SymbolInfo => info.SymbolInfo; + + public IMethodSymbol TargetSymbol + { + get + { + var si = SymbolInfo; + + if (si.Symbol != null) return si.Symbol as IMethodSymbol; + + if (si.CandidateReason == CandidateReason.OverloadResolutionFailure) + { + // This seems to be a bug in Roslyn + // For some reason, typeof(X).InvokeMember(...) fails to resolve the correct + // InvokeMember() method, even though the number of parameters clearly identifies the correct method + + var candidates = si.CandidateSymbols. + OfType(). + Where(method => method.Parameters.Length >= Syntax.ArgumentList.Arguments.Count). + Where(method => method.Parameters.Count(p => !p.HasExplicitDefaultValue) <= Syntax.ArgumentList.Arguments.Count); + + return cx.Extractor.Standalone ? + candidates.FirstOrDefault() : + candidates.SingleOrDefault(); + } + + return si.Symbol as IMethodSymbol; + } + } + + static bool IsDelegateCall(ExpressionNodeInfo info) + { + var si = info.SymbolInfo; + + if (si.CandidateReason == CandidateReason.OverloadResolutionFailure && + si.CandidateSymbols.OfType().All(s => s.MethodKind == MethodKind.DelegateInvoke)) + return true; + + // Delegate variable is a dynamic + var node = (InvocationExpressionSyntax)info.Node; + if (si.CandidateReason == CandidateReason.LateBound && + node.Expression is IdentifierNameSyntax && + IsDynamic(info.Context, node.Expression) && + si.Symbol == null) + return true; + + return si.Symbol != null && + si.Symbol.Kind == SymbolKind.Method && + ((IMethodSymbol)si.Symbol).MethodKind == MethodKind.DelegateInvoke; + } + + static bool IsLocalFunctionInvocation(ExpressionNodeInfo info) + { + var target = info.SymbolInfo.Symbol as IMethodSymbol; + return target != null && target.MethodKind == MethodKind.LocalFunction; + } + + static ExprKind GetKind(ExpressionNodeInfo info) + { + return IsNameof((InvocationExpressionSyntax)info.Node) ? + ExprKind.NAMEOF : + IsDelegateCall(info) ? + ExprKind.DELEGATE_INVOCATION : + IsLocalFunctionInvocation(info) ? + ExprKind.LOCAL_FUNCTION_INVOCATION : + ExprKind.METHOD_INVOCATION; + } + + static bool IsNameof(InvocationExpressionSyntax syntax) + { + // Odd that this is not a separate expression type. + // Maybe it will be in the future. + var id = syntax.Expression as IdentifierNameSyntax; + return id != null && id.Identifier.Text == "nameof"; + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/IsPattern.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/IsPattern.cs new file mode 100644 index 00000000000..e26605931ca --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/IsPattern.cs @@ -0,0 +1,46 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Microsoft.CodeAnalysis.CSharp; +using Semmle.Extraction.Kinds; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class IsPattern : Expression + { + IsPattern(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.IS)) + { + } + + protected override void Populate() + { + var constantPattern = Syntax.Pattern as ConstantPatternSyntax; + if (constantPattern != null) + { + Create(cx, Syntax.Expression, this, 0); + Create(cx, constantPattern.Expression, this, 3); + return; + } + + var pattern = Syntax.Pattern as DeclarationPatternSyntax; + + if (pattern == null) + { + throw new InternalError(Syntax, "Is-pattern not handled"); + } + + Create(cx, Syntax.Expression, this, 0); + TypeAccess.Create(cx, pattern.Type, this, 1); + + var symbol = cx.Model(Syntax).GetDeclaredSymbol(pattern.Designation) as ILocalSymbol; + if (symbol != null) + { + var type = Type.Create(cx, symbol.Type); + var isVar = pattern.Type.IsVar; + VariableDeclaration.Create(cx, symbol, type, cx.Create(pattern.GetLocation()), cx.Create(pattern.Designation.GetLocation()), isVar, this, 2); + } + } + + public static Expression Create(ExpressionNodeInfo info) => new IsPattern(info).TryPopulate(); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Lambda.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Lambda.cs new file mode 100644 index 00000000000..3a82004c5b7 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Lambda.cs @@ -0,0 +1,56 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis; +using System.Collections.Generic; +using Semmle.Util; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Lambda : Expression, IStatementParentEntity + { + bool IStatementParentEntity.IsTopLevelParent => false; + + protected override void Populate() { } + + void VisitParameter(Context cx, ParameterSyntax p) + { + var symbol = cx.Model(p).GetDeclaredSymbol(p); + Parameter.Create(cx, symbol, this); + } + + Lambda(ExpressionNodeInfo info, CSharpSyntaxNode body, IEnumerable @params) + : base(info) + { + // No need to use `Populate` as the population happens later + cx.PopulateLater(() => + { + foreach (var param in @params) + VisitParameter(cx, param); + + if (body is ExpressionSyntax) + Create(cx, (ExpressionSyntax)body, this, 0); + else if (body is BlockSyntax) + Statements.Block.Create(cx, (BlockSyntax)body, this, 0); + else + cx.ModelError(body, "Unhandled lambda body"); + }); + } + + Lambda(ExpressionNodeInfo info, ParenthesizedLambdaExpressionSyntax node) + : this(info.SetKind(ExprKind.LAMBDA), node.Body, node.ParameterList.Parameters) { } + + public static Lambda Create(ExpressionNodeInfo info, ParenthesizedLambdaExpressionSyntax node) => new Lambda(info, node); + + Lambda(ExpressionNodeInfo info, SimpleLambdaExpressionSyntax node) + : this(info.SetKind(ExprKind.LAMBDA), node.Body, Enumerators.Singleton(node.Parameter)) { } + + public static Lambda Create(ExpressionNodeInfo info, SimpleLambdaExpressionSyntax node) => new Lambda(info, node); + + Lambda(ExpressionNodeInfo info, AnonymousMethodExpressionSyntax node) : + this(info.SetKind(ExprKind.ANONYMOUS_METHOD), node.Body, node.ParameterList == null ? Enumerable.Empty() : node.ParameterList.Parameters) { } + + public static Lambda Create(ExpressionNodeInfo info, AnonymousMethodExpressionSyntax node) => new Lambda(info, node); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Literal.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Literal.cs new file mode 100644 index 00000000000..5dcc6564dfe --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Literal.cs @@ -0,0 +1,72 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis; +using Semmle.Extraction.CSharp.Populators; +using Microsoft.CodeAnalysis.CSharp; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Literal : Expression + { + Literal(ExpressionNodeInfo info) : base(info.SetKind(GetKind(info)) ) { } + + public static Expression Create(ExpressionNodeInfo info) => new Literal(info).TryPopulate(); + + protected override void Populate() { } + + static ExprKind GetKind(ExpressionNodeInfo info) + { + switch(info.Node.Kind()) + { + case SyntaxKind.DefaultLiteralExpression: + return ExprKind.DEFAULT; + case SyntaxKind.NullLiteralExpression: + info.Type = Type.Create(info.Context, null); // Don't use converted type. + return ExprKind.NULL_LITERAL; + } + + var type = info.Type.symbol; + + switch (type.SpecialType) + { + case SpecialType.System_Boolean: + return ExprKind.BOOL_LITERAL; + + case SpecialType.System_Int16: + case SpecialType.System_Int32: + case SpecialType.System_Byte: // ?? + return ExprKind.INT_LITERAL; + + case SpecialType.System_Char: + return ExprKind.CHAR_LITERAL; + + case SpecialType.System_Decimal: + return ExprKind.DECIMAL_LITERAL; + + case SpecialType.System_Double: + return ExprKind.DOUBLE_LITERAL; + + case SpecialType.System_Int64: + return ExprKind.LONG_LITERAL; + + case SpecialType.System_Single: + return ExprKind.FLOAT_LITERAL; + + case SpecialType.System_String: + return ExprKind.STRING_LITERAL; + + case SpecialType.System_UInt16: + case SpecialType.System_UInt32: + case SpecialType.System_SByte: // ?? + return ExprKind.UINT_LITERAL; + + case SpecialType.System_UInt64: + return ExprKind.ULONG_LITERAL; + + default: + info.Context.ModelError(info.Node, "Unhandled literal type"); + return ExprKind.UNKNOWN; + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MakeRef.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MakeRef.cs new file mode 100644 index 00000000000..168f413ae82 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MakeRef.cs @@ -0,0 +1,17 @@ +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis.CSharp.Syntax; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class MakeRef : Expression + { + MakeRef(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.REF)) { } + + public static Expression Create(ExpressionNodeInfo info) => new MakeRef(info).TryPopulate(); + + protected override void Populate() + { + Create(cx, Syntax.Expression, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MemberAccess.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MemberAccess.cs new file mode 100644 index 00000000000..7b378f83210 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MemberAccess.cs @@ -0,0 +1,101 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class MemberAccess : Expression + { + private MemberAccess(ExpressionNodeInfo info, ExpressionSyntax qualifier, ISymbol target) : base(info) + { + Qualifier = Create(cx, qualifier, this, -1); + + if (target == null) + { + if (info.Kind != ExprKind.DYNAMIC_MEMBER_ACCESS) + cx.ModelError(info.Node, "Could not determine target for member access"); + } + else + { + cx.Emit(Tuples.expr_access(this, cx.CreateEntity(target))); + } + } + + public static Expression Create(ExpressionNodeInfo info, ConditionalAccessExpressionSyntax node) => + // The qualifier is located by walking the syntax tree. + // `node.WhenNotNull` will contain a MemberBindingExpressionSyntax, calling the method below. + Create(info.Context, node.WhenNotNull, info.Parent, info.Child); + + public static Expression Create(ExpressionNodeInfo info, MemberBindingExpressionSyntax node) + { + var expr = Create(info, FindConditionalQualifier(node), node.Name); + expr.MakeConditional(); + return expr; + } + + public static Expression Create(ExpressionNodeInfo info, MemberAccessExpressionSyntax node) => + Create(info, node.Expression, node.Name); + + static Expression Create(ExpressionNodeInfo info, ExpressionSyntax expression, SimpleNameSyntax name) + { + if (IsDynamic(info.Context, expression)) + { + var expr = new MemberAccess(info.SetKind(ExprKind.DYNAMIC_MEMBER_ACCESS), expression, null); + info.Context.Emit(Tuples.dynamic_member_name(expr, name.Identifier.Text)); + return expr; + } + + var target = info.SymbolInfo; + + if (target.CandidateReason == CandidateReason.OverloadResolutionFailure) + { + // Roslyn workaround. Even if we can't resolve a method, we know it's a method. + return Create(info.Context, expression, info.Parent, info.Child); + } + + var symbol = target.Symbol ?? info.Context.GetSymbolInfo(name).Symbol; + + if (symbol == null && target.CandidateSymbols.Length >= 1) + { + // Pick the first symbol. This could occur for something like `nameof(Foo.Bar)` + // where `Bar` is a method group. Technically, we don't know which symbol is accessed. + symbol = target.CandidateSymbols[0]; + } + + if (symbol == null) + { + info.Context.ModelError(info.Node, "Failed to determine symbol for member access"); + return new MemberAccess(info.SetKind(ExprKind.UNKNOWN), expression, symbol); + } + + ExprKind kind; + + switch (symbol.Kind) + { + case SymbolKind.Property: + kind = ExprKind.PROPERTY_ACCESS; + break; + case SymbolKind.Method: + kind = ExprKind.METHOD_ACCESS; + break; + case SymbolKind.Field: + kind = ExprKind.FIELD_ACCESS; + break; + case SymbolKind.NamedType: + return TypeAccess.Create(info); + case SymbolKind.Event: + kind = ExprKind.EVENT_ACCESS; + break; + default: + info.Context.ModelError(info.Node, "Unhandled symbol for member access"); + kind = ExprKind.UNKNOWN; + break; + } + return new MemberAccess(info.SetKind(kind), expression, symbol); + } + + public Expression Qualifier { get; private set; } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Name.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Name.cs new file mode 100644 index 00000000000..c76753789ad --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Name.cs @@ -0,0 +1,66 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + static class Name + { + public static Expression Create(ExpressionNodeInfo info) + { + var symbolInfo = info.Context.GetSymbolInfo(info.Node); + + var target = symbolInfo.Symbol; + + if (target == null && symbolInfo.CandidateReason == CandidateReason.OverloadResolutionFailure) + { + // The expression is probably a cast + target = info.Context.GetSymbolInfo((CSharpSyntaxNode)info.Node.Parent).Symbol; + } + + if (target == null && (symbolInfo.CandidateReason == CandidateReason.Ambiguous || symbolInfo.CandidateReason == CandidateReason.MemberGroup)) + { + // Pick one at random - they probably resolve to the same ID + target = symbolInfo.CandidateSymbols.First(); + } + + if (target == null) + { + info.Context.ModelError(info.Node, "Failed to resolve name"); + return new Unknown(info); + } + + // There is a very strange bug in Microsoft.CodeAnalysis whereby + // target.Kind throws System.InvalidOperationException for Discard symbols. + // So, short-circuit that test here. + // Ideally this would be another case in the switch statement below. + if (target is IDiscardSymbol) + return new Discard(info); + + switch (target.Kind) + { + case SymbolKind.TypeParameter: + case SymbolKind.NamedType: + return TypeAccess.Create(info); + + case SymbolKind.Property: + case SymbolKind.Field: + case SymbolKind.Event: + case SymbolKind.Method: + return Access.Create(info, target, true, info.Context.CreateEntity(target)); + + case SymbolKind.Local: + case SymbolKind.RangeVariable: + return Access.Create(info, target, false, LocalVariable.GetAlreadyCreated(info.Context, target)); + + case SymbolKind.Parameter: + return Access.Create(info, target, false, Parameter.GetAlreadyCreated(info.Context, (IParameterSymbol)target)); + + default: + throw new InternalError(info.Node, "Unhandled identifier kind '{0}'", target.Kind); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ObjectCreation.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ObjectCreation.cs new file mode 100644 index 00000000000..e90d04df200 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ObjectCreation.cs @@ -0,0 +1,136 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Entities; +using Semmle.Extraction.Kinds; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + abstract class ObjectCreation : Expression where SyntaxNode : ExpressionSyntax + { + protected ObjectCreation(ExpressionNodeInfo info) + : base(info) { } + } + + // new Foo(...) { ... }. + class ExplicitObjectCreation : ObjectCreation + { + static bool IsDynamicObjectCreation(Context cx, ObjectCreationExpressionSyntax node) + { + return node.ArgumentList != null && node.ArgumentList.Arguments.Any(arg => IsDynamic(cx, arg.Expression)); + } + + static ExprKind GetKind(Context cx, ObjectCreationExpressionSyntax node) + { + var si = cx.Model(node).GetSymbolInfo(node.Type); + return Type.IsDelegate(si.Symbol as INamedTypeSymbol) ? ExprKind.EXPLICIT_DELEGATE_CREATION : ExprKind.OBJECT_CREATION; + } + + ExplicitObjectCreation(ExpressionNodeInfo info) + : base(info.SetKind(GetKind(info.Context, (ObjectCreationExpressionSyntax)info.Node))) { } + + public static Expression Create(ExpressionNodeInfo info) => new ExplicitObjectCreation(info).TryPopulate(); + + protected override void Populate() + { + if (Syntax.ArgumentList != null) + { + PopulateArguments(cx, Syntax.ArgumentList, 0); + } + + var target = cx.Model(Syntax).GetSymbolInfo(Syntax); + var method = (IMethodSymbol)target.Symbol; + + if (method != null) + { + cx.Emit(Tuples.expr_call(this, Method.Create(cx, method))); + } + + if (IsDynamicObjectCreation(cx, Syntax)) + { + var name = GetDynamicName(Syntax.Type); + if (name.HasValue) + cx.Emit(Tuples.dynamic_member_name(this, name.Value.Text)); + else + cx.ModelError(Syntax, "Unable to get name for dynamic object creation."); + } + + if (Syntax.Initializer != null) + { + switch (Syntax.Initializer.Kind()) + { + case SyntaxKind.CollectionInitializerExpression: + CollectionInitializer.Create(new ExpressionNodeInfo(cx, Syntax.Initializer, this, -1) { Type = Type }); + break; + case SyntaxKind.ObjectInitializerExpression: + ObjectInitializer.Create(new ExpressionNodeInfo(cx, Syntax.Initializer, this, -1) { Type = Type }); + break; + default: + cx.ModelError("Unhandled initializer in object creation"); + break; + } + } + + TypeMention.Create(cx, Syntax.Type, this, Type); + } + + static SyntaxToken? GetDynamicName(CSharpSyntaxNode name) + { + switch (name.Kind()) + { + case SyntaxKind.IdentifierName: + return ((IdentifierNameSyntax)name).Identifier; + case SyntaxKind.GenericName: + return ((GenericNameSyntax)name).Identifier; + case SyntaxKind.QualifiedName: + // We ignore any qualifiers, for now + return GetDynamicName(((QualifiedNameSyntax)name).Right); + default: + return null; + } + } + } + + class ImplicitObjectCreation : ObjectCreation + { + public ImplicitObjectCreation(ExpressionNodeInfo info) + : base(info.SetKind(ExprKind.OBJECT_CREATION)) { } + + public static Expression Create(ExpressionNodeInfo info) => + new ImplicitObjectCreation(info).TryPopulate(); + + protected override void Populate() + { + var target = cx.GetSymbolInfo(Syntax); + var method = (IMethodSymbol)target.Symbol; + + if (method != null) + { + cx.Emit(Tuples.expr_call(this, Method.Create(cx, method))); + } + var child = 0; + + Expression objectInitializer = Syntax.Initializers.Any() ? + new Expression(new ExpressionInfo(cx, Type, Location, ExprKind.OBJECT_INIT, this, -1, false, null)) : + null; + + foreach (var init in Syntax.Initializers) + { + // Create an "assignment" + var property = cx.Model(init).GetDeclaredSymbol(init); + var propEntity = Property.Create(cx, property); + var type = Type.Create(cx, property.Type); + var loc = cx.Create(init.GetLocation()); + + var assignment = new Expression(new ExpressionInfo(cx, type, loc, ExprKind.SIMPLE_ASSIGN, objectInitializer, child++, false, null)); + Create(cx, init.Expression, assignment, 0); + Property.Create(cx, property); + + var access = new Expression(new ExpressionInfo(cx, type, loc, ExprKind.PROPERTY_ACCESS, assignment, 1, false, null)); + cx.Emit(Tuples.expr_access(access, propEntity)); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/PointerMemberAccess.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/PointerMemberAccess.cs new file mode 100644 index 00000000000..6087c9c4df2 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/PointerMemberAccess.cs @@ -0,0 +1,19 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class PointerMemberAccess : Expression + { + PointerMemberAccess(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.POINTER_INDIRECTION)) { } + + public static Expression Create(ExpressionNodeInfo info) => new PointerMemberAccess(info).TryPopulate(); + + protected override void Populate() + { + Create(cx, Syntax.Expression, this, 0); + + // !! We do not currently look at the member (or store the member name). + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/PostfixUnary.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/PostfixUnary.cs new file mode 100644 index 00000000000..a596ab93c43 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/PostfixUnary.cs @@ -0,0 +1,32 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class PostfixUnary : Expression + { + PostfixUnary(ExpressionNodeInfo info, ExprKind kind, ExpressionSyntax operand) + : base(info.SetKind(UnaryOperatorKind(info.Context, kind, info.Node))) + { + Operand = operand; + OperatorKind = kind; + } + + readonly ExpressionSyntax Operand; + readonly ExprKind OperatorKind; + + public static Expression Create(ExpressionNodeInfo info, ExpressionSyntax operand) => new PostfixUnary(info, info.Kind, operand).TryPopulate(); + + protected override void Populate() + { + Create(cx, Operand, this, 0); + OperatorCall(Syntax); + + if ((OperatorKind == ExprKind.POST_INCR || OperatorKind == ExprKind.POST_DECR) && + Kind == ExprKind.OPERATOR_INVOCATION) + { + cx.Emit(Tuples.mutator_invocation_mode(this, 2)); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Query.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Query.cs new file mode 100644 index 00000000000..d5c8ead52fd --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Query.cs @@ -0,0 +1,260 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using System.Linq; +using Semmle.Extraction.Kinds; +using System.Collections.Generic; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + static class Query + { + /// + /// An expression representing a call in a LINQ query. + /// + /// + /// + /// This code has some problems because the expression kind isn't correct - it should really be + /// a new ExprKind. The other issue is that the argument to the call is technically a lambda, + /// rather than the sub-expression itself. + /// + private class QueryCall : Expression + { + public QueryCall(Context cx, IMethodSymbol method, SyntaxNode clause, IExpressionParentEntity parent, int child) + : base(new ExpressionInfo(cx, Type.Create(cx, method?.ReturnType), cx.Create(clause.GetLocation()), ExprKind.METHOD_INVOCATION, parent, child, false, null)) + { + if (method != null) + cx.Emit(Tuples.expr_call(this, Method.Create(cx, method))); + } + } + + /// + /// Represents a chain of method calls (the operand being recursive). + /// + class ClauseCall + { + public ClauseCall operand; + public IMethodSymbol method; + public readonly List arguments = new List(); + public SyntaxNode node; + public ISymbol declaration; + public SyntaxToken name; + public ISymbol intoDeclaration; + + public ExpressionSyntax Expr => arguments.First(); + + public ClauseCall WithClause(IMethodSymbol newMethod, SyntaxNode newNode, SyntaxToken newName = default(SyntaxToken), ISymbol newDeclaration = null) + { + return new ClauseCall + { + operand = this, + method = newMethod, + node = newNode, + name = newName, + declaration = newDeclaration + }; + } + + public ClauseCall AddArgument(ExpressionSyntax arg) + { + if (arg != null) + arguments.Add(arg); + return this; + } + + public ClauseCall WithInto(ISymbol into) + { + intoDeclaration = into; + return this; + } + + Expression DeclareRangeVariable(Context cx, IExpressionParentEntity parent, int child, bool getElement) + { + return DeclareRangeVariable(cx, parent, child, getElement, declaration); + } + + void DeclareIntoVariable(Context cx, IExpressionParentEntity parent, int intoChild, bool getElement) + { + if (intoDeclaration != null) + DeclareRangeVariable(cx, parent, intoChild, getElement, intoDeclaration); + } + + Expression DeclareRangeVariable(Context cx, IExpressionParentEntity parent, int child, bool getElement, ISymbol variableSymbol) + { + var type = Type.Create(cx, cx.GetType(Expr)); + Semmle.Extraction.Entities.Location nameLoc; + + Type declType; + if (getElement) + { + var from = node as FromClauseSyntax; + declType = from != null && from.Type != null + ? Type.Create(cx, cx.GetType(from.Type)) + : type.ElementType; + } + else + declType = type; + + var decl = VariableDeclaration.Create(cx, + variableSymbol, + declType, + cx.Create(node.GetLocation()), + nameLoc = cx.Create(name.GetLocation()), + true, + parent, + child + ); + + Expression.Create(cx, Expr, decl, 0); + + var access = new Expression(new ExpressionInfo(cx, type, nameLoc, ExprKind.LOCAL_VARIABLE_ACCESS, decl, 1, false, null)); + cx.Emit(Tuples.expr_access(access, LocalVariable.GetAlreadyCreated(cx, variableSymbol))); + + return decl; + } + + void PopulateArguments(Context cx, QueryCall callExpr, int child) + { + foreach (var e in arguments) + { + Expression.Create(cx, e, callExpr, child++); + } + } + + public Expression Populate(Context cx, IExpressionParentEntity parent, int child) + { + if (declaration != null) // The first "from" clause, or a "let" clause + { + if (operand == null) + { + return DeclareRangeVariable(cx, parent, child, true); + } + else + { + if (method == null) + cx.ModelError(node, "Unable to determine target of query expression"); + + var callExpr = new QueryCall(cx, method, node, parent, child); + operand.Populate(cx, callExpr, 0); + DeclareRangeVariable(cx, callExpr, 1, false); + PopulateArguments(cx, callExpr, 2); + DeclareIntoVariable(cx, callExpr, 2 + arguments.Count, false); + return callExpr; + } + } + else + { + var callExpr = new QueryCall(cx, method, node, parent, child); + operand.Populate(cx, callExpr, 0); + PopulateArguments(cx, callExpr, 1); + return callExpr; + } + } + } + + /// + /// Construct a "syntax tree" representing the LINQ query. + /// + /// + /// The extraction context. + /// The query expression. + /// A "syntax tree" of the query. + static ClauseCall ConstructQueryExpression(Context cx, QueryExpressionSyntax node) + { + var info = cx.Model(node).GetQueryClauseInfo(node.FromClause); + var method = info.OperationInfo.Symbol as IMethodSymbol; + + ClauseCall clauseExpr = new ClauseCall + { + declaration = cx.Model(node).GetDeclaredSymbol(node.FromClause), + name = node.FromClause.Identifier, + method = method, + node = node.FromClause + }.AddArgument(node.FromClause.Expression); + + foreach (var qc in node.Body.Clauses) + { + info = cx.Model(node).GetQueryClauseInfo(qc); + + method = info.OperationInfo.Symbol as IMethodSymbol; + + switch (qc.Kind()) + { + case SyntaxKind.OrderByClause: + var orderByClause = (OrderByClauseSyntax)qc; + foreach (var ordering in orderByClause.Orderings) + { + method = cx.Model(node).GetSymbolInfo(ordering).Symbol as IMethodSymbol; + + clauseExpr = clauseExpr.WithClause(method, orderByClause).AddArgument(ordering.Expression); + + if (method == null) + cx.ModelError(ordering, "Could not determine method call for orderby clause"); + } + break; + case SyntaxKind.WhereClause: + var whereClause = (WhereClauseSyntax)qc; + clauseExpr = clauseExpr.WithClause(method, whereClause).AddArgument(whereClause.Condition); + break; + case SyntaxKind.FromClause: + var fromClause = (FromClauseSyntax)qc; + clauseExpr = clauseExpr. + WithClause(method, fromClause, fromClause.Identifier, cx.Model(node).GetDeclaredSymbol(fromClause)). + AddArgument(fromClause.Expression); + break; + case SyntaxKind.LetClause: + var letClause = (LetClauseSyntax)qc; + clauseExpr = clauseExpr.WithClause(method, letClause, letClause.Identifier, cx.Model(node).GetDeclaredSymbol(letClause)). + AddArgument(letClause.Expression); + break; + case SyntaxKind.JoinClause: + var joinClause = (JoinClauseSyntax)qc; + + clauseExpr = clauseExpr.WithClause(method, joinClause, joinClause.Identifier, cx.Model(node).GetDeclaredSymbol(joinClause)). + AddArgument(joinClause.InExpression). + AddArgument(joinClause.LeftExpression). + AddArgument(joinClause.RightExpression); + + if (joinClause.Into != null) + { + var into = cx.Model(node).GetDeclaredSymbol(joinClause.Into); + clauseExpr.WithInto(into); + } + + break; + default: + throw new InternalError(qc, "Unhandled query clause of kind {0}", qc.Kind()); + } + } + + method = cx.Model(node).GetSymbolInfo(node.Body.SelectOrGroup).Symbol as IMethodSymbol; + + var selectClause = node.Body.SelectOrGroup as SelectClauseSyntax; + var groupClause = node.Body.SelectOrGroup as GroupClauseSyntax; + + clauseExpr = new ClauseCall { operand = clauseExpr, method = method, node = node.Body.SelectOrGroup }; + + if (selectClause != null) + { + clauseExpr.AddArgument(selectClause.Expression); + } + else if (groupClause != null) + { + clauseExpr. + AddArgument(groupClause.GroupExpression). + AddArgument(groupClause.ByExpression); + } + else + { + throw new InternalError(node, "Failed to process select/group by clause"); + } + + return clauseExpr; + } + + public static Expression Create(ExpressionNodeInfo info) => + ConstructQueryExpression(info.Context, (QueryExpressionSyntax)info.Node).Populate(info.Context, info.Parent, info.Child); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Ref.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Ref.cs new file mode 100644 index 00000000000..a5e4e905970 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Ref.cs @@ -0,0 +1,17 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Ref : Expression + { + Ref(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.REF)) { } + + public static Expression Create(ExpressionNodeInfo info) => new Ref(info).TryPopulate(); + + protected override void Populate() + { + Create(cx, Syntax.Expression, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/RefType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/RefType.cs new file mode 100644 index 00000000000..d1190f5d7e0 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/RefType.cs @@ -0,0 +1,17 @@ +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis.CSharp.Syntax; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class RefType : Expression + { + RefType(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.UNKNOWN)) { } + + public static Expression Create(ExpressionNodeInfo info) => new RefType(info).TryPopulate(); + + protected override void Populate() + { + Create(cx, Syntax.Expression, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/RefValue.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/RefValue.cs new file mode 100644 index 00000000000..df3ee364346 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/RefValue.cs @@ -0,0 +1,18 @@ +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis.CSharp.Syntax; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class RefValue : Expression + { + RefValue(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.REF)) { } + + public static Expression Create(ExpressionNodeInfo info) => new RefValue(info).TryPopulate(); + + protected override void Populate() + { + Create(cx, Syntax.Expression, this, 0); + Create(cx, Syntax.Type, this, 1); // A type-access + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Sizeof.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Sizeof.cs new file mode 100644 index 00000000000..095b4672f2d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Sizeof.cs @@ -0,0 +1,17 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class SizeOf : Expression + { + SizeOf(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.SIZEOF)) { } + + public static Expression Create(ExpressionNodeInfo info) => new SizeOf(info).TryPopulate(); + + protected override void Populate() + { + TypeAccess.Create(cx, Syntax.Type, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/This.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/This.cs new file mode 100644 index 00000000000..cc9514cd639 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/This.cs @@ -0,0 +1,16 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Entities; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class This : Expression + { + This(IExpressionInfo info) : base(info) { } + + public static This CreateImplicit(Context cx, Type @class, Location loc, IExpressionParentEntity parent, int child) => + new This(new ExpressionInfo(cx, @class, loc, Kinds.ExprKind.THIS_ACCESS, parent, child, true, null)); + + public static This CreateExplicit(ExpressionNodeInfo info) => new This(info.SetKind(ExprKind.THIS_ACCESS)); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Throw.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Throw.cs new file mode 100644 index 00000000000..5f365fc82da --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Throw.cs @@ -0,0 +1,17 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Throw : Expression + { + Throw(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.THROW)) { } + + public static Expression Create(ExpressionNodeInfo info) => new Throw(info).TryPopulate(); + + protected override void Populate() + { + Create(cx, Syntax.Expression, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Tuple.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Tuple.cs new file mode 100644 index 00000000000..7fa86d88ebc --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Tuple.cs @@ -0,0 +1,24 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Tuple : Expression + { + public static Expression Create(ExpressionNodeInfo info) => new Tuple(info).TryPopulate(); + + Tuple(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.TUPLE)) + { + } + + protected override void Populate() + { + int child = 0; + foreach (var argument in Syntax.Arguments.Select(a => a.Expression)) + { + Expression.Create(cx, argument, this, child++); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/TypeAccess.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/TypeAccess.cs new file mode 100644 index 00000000000..ee6997b6431 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/TypeAccess.cs @@ -0,0 +1,38 @@ +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class TypeAccess : Expression + { + TypeAccess(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.TYPE_ACCESS)) { } + + protected override void Populate() + { + switch (Syntax.Kind()) + { + case SyntaxKind.SimpleMemberAccessExpression: + var maes = (MemberAccessExpressionSyntax)Syntax; + if (Type.ContainingType == null) + { + // namespace qualifier + TypeMention.Create(cx, maes.Name, this, Type, Syntax.GetLocation()); + } + else + { + // type qualifier + TypeMention.Create(cx, maes.Name, this, Type); + Create(cx, maes.Expression, this, -1); + } + return; + default: + TypeMention.Create(cx, (TypeSyntax)Syntax, this, Type); + return; + } + } + + public static Expression Create(ExpressionNodeInfo info) => new TypeAccess(info).TryPopulate(); + } +} + diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/TypeOf.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/TypeOf.cs new file mode 100644 index 00000000000..f343614bfde --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/TypeOf.cs @@ -0,0 +1,17 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class TypeOf : Expression + { + TypeOf(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.TYPEOF)) { } + + public static Expression Create(ExpressionNodeInfo info) => new TypeOf(info).TryPopulate(); + + protected override void Populate() + { + TypeAccess.Create(cx, Syntax.Type, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Unary.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Unary.cs new file mode 100644 index 00000000000..6f518477e2a --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Unary.cs @@ -0,0 +1,35 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Unary : Expression + { + Unary(ExpressionNodeInfo info, ExprKind kind) + : base(info.SetKind(UnaryOperatorKind(info.Context, info.Kind, info.Node))) + { + OperatorKind = kind; + } + + readonly ExprKind OperatorKind; + + public static Unary Create(ExpressionNodeInfo info) + { + var ret = new Unary(info, info.Kind); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + Create(cx, Syntax.Operand, this, 0); + OperatorCall(Syntax); + + if ((OperatorKind == ExprKind.PRE_INCR || OperatorKind == ExprKind.PRE_DECR) && + Kind == ExprKind.OPERATOR_INVOCATION) + { + cx.Emit(Tuples.mutator_invocation_mode(this, 1)); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Unchecked.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Unchecked.cs new file mode 100644 index 00000000000..73ec785cd9f --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Unchecked.cs @@ -0,0 +1,17 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Unchecked : Expression + { + Unchecked(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.UNCHECKED)) { } + + public static Expression Create(ExpressionNodeInfo info) => new Unchecked(info).TryPopulate(); + + protected override void Populate() + { + Create(cx, Syntax.Expression, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Unknown.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Unknown.cs new file mode 100644 index 00000000000..c547626e639 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Unknown.cs @@ -0,0 +1,10 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Unknown : Expression + { + public Unknown(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.UNKNOWN)) { } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/VariableDeclaration.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/VariableDeclaration.cs new file mode 100644 index 00000000000..70d6cdfe072 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/VariableDeclaration.cs @@ -0,0 +1,145 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Kinds; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class VariableDeclaration : Expression + { + VariableDeclaration(IExpressionInfo info) : base(info) { } + + public static VariableDeclaration Create(Context cx, ISymbol symbol, Type type, Extraction.Entities.Location exprLocation, Extraction.Entities.Location declLocation, bool isVar, IExpressionParentEntity parent, int child) + { + var ret = new VariableDeclaration(new ExpressionInfo(cx, type, exprLocation, ExprKind.LOCAL_VAR_DECL, parent, child, false, null)); + cx.Try(null, null, () => LocalVariable.Create(cx, symbol, ret, isVar, declLocation)); + return ret; + } + + static VariableDeclaration CreateSingle(Context cx, DeclarationExpressionSyntax node, SingleVariableDesignationSyntax designation, IExpressionParentEntity parent, int child) + { + bool isVar = node.Type.IsVar; + + var variableSymbol = cx.Model(designation).GetDeclaredSymbol(designation) as ILocalSymbol; + if (variableSymbol == null) + { + cx.ModelError(node, "Failed to determine local variable"); + return new VariableDeclaration(cx, node, null, isVar, parent, child); + } + + var type = Type.Create(cx, variableSymbol.Type); + var location = cx.Create(designation.GetLocation()); + + var ret = new VariableDeclaration(cx, designation, type, isVar, parent, child); + cx.Try(null, null, () => LocalVariable.Create(cx, variableSymbol, ret, isVar, location)); + return ret; + } + + /// + /// Create a tuple expression representing a parenthesized variable declaration. + /// That is, we consider `var (x, y) = ...` to be equivalent to `(var x, var y) = ...`. + /// + static Expression CreateParenthesized(Context cx, DeclarationExpressionSyntax node, ParenthesizedVariableDesignationSyntax designation, IExpressionParentEntity parent, int child) + { + var type = Type.Create(cx, null); // Should ideally be a corresponding tuple type + var tuple = new Expression(new ExpressionInfo(cx, type, cx.Create(node.GetLocation()), ExprKind.TUPLE, parent, child, false, null)); + + cx.Try(null, null, () => + { + var child0 = 0; + foreach (var variable in designation.Variables) + Create(cx, node, variable, tuple, child0++); + }); + + return tuple; + } + + static Expression Create(Context cx, DeclarationExpressionSyntax node, VariableDesignationSyntax designation, IExpressionParentEntity parent, int child) + { + var single = designation as SingleVariableDesignationSyntax; + if (single != null) + return CreateSingle(cx, node, single, parent, child); + + var paren = designation as ParenthesizedVariableDesignationSyntax; + if (paren != null) + return CreateParenthesized(cx, node, paren, parent, child); + + var discard = designation as DiscardDesignationSyntax; + if (discard != null) + { + var type = cx.Model(node).GetTypeInfo(node).Type; + return new VariableDeclaration(cx, node, Type.Create(cx, type), node.Type.IsVar, parent, child); + } + + cx.ModelError(node, "Failed to determine designation type"); + return new VariableDeclaration(cx, node, null, node.Type.IsVar, parent, child); + } + + public static Expression Create(Context cx, DeclarationExpressionSyntax node, IExpressionParentEntity parent, int child) + { + return Create(cx, node, node.Designation, parent, child); + } + + VariableDeclaration(Context cx, CSharpSyntaxNode d, Type type, bool isVar, IExpressionParentEntity parent, int child) + : base(new ExpressionInfo(cx, type, cx.Create(d.FixedLocation()), ExprKind.LOCAL_VAR_DECL, parent, child, false, null)) + { + } + + public static VariableDeclaration Create(Context cx, CatchDeclarationSyntax d, bool isVar, IExpressionParentEntity parent, int child) + { + var type = Type.Create(cx, cx.Model(d).GetDeclaredSymbol(d).Type); + var ret = new VariableDeclaration(cx, d, type, isVar, parent, child); + cx.Try(d, null, () => + { + var id = d.Identifier; + var declSymbol = cx.Model(d).GetDeclaredSymbol(d); + var location = cx.Create(id.GetLocation()); + LocalVariable.Create(cx, declSymbol, ret, isVar, location); + TypeMention.Create(cx, d.Type, ret, type); + }); + return ret; + } + + public static VariableDeclaration Create(Context cx, VariableDeclaratorSyntax d, Type type, bool isVar, IExpressionParentEntity parent, int child) + { + var ret = new VariableDeclaration(cx, d, type, isVar, parent, child); + cx.Try(d, null, () => + { + var id = d.Identifier; + var declSymbol = cx.Model(d).GetDeclaredSymbol(d); + var location = cx.Create(id.GetLocation()); + var localVar = LocalVariable.Create(cx, declSymbol, ret, isVar, location); + + if (d.Initializer != null) + { + Create(cx, d.Initializer.Value, ret, 0); + + // Create an access + var access = new Expression(new ExpressionInfo(cx, type, location, ExprKind.LOCAL_VARIABLE_ACCESS, ret, 1, false, null)); + cx.Emit(Tuples.expr_access(access, localVar)); + } + + var decl = d.Parent as VariableDeclarationSyntax; + if (decl != null) + TypeMention.Create(cx, decl.Type, ret, type); + }); + return ret; + } + } + + static class VariableDeclarations + { + public static void Populate(Context cx, VariableDeclarationSyntax decl, IExpressionParentEntity parent, int child, int childIncrement = 1) + { + var type = Type.Create(cx, cx.GetType(decl.Type)); + + foreach (var v in decl.Variables) + { + VariableDeclaration.Create(cx, v, type, decl.Type.IsVar, parent, child); + child += childIncrement; + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs new file mode 100644 index 00000000000..541d49968e6 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs @@ -0,0 +1,101 @@ +using Microsoft.CodeAnalysis; +using Semmle.Extraction.CSharp.Populators; +using System.Linq; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using System; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities +{ + class Field : CachedSymbol, IExpressionParentEntity + { + Field(Context cx, IFieldSymbol init) + : base(cx, init) + { + type = new Lazy(() => Type.Create(cx, symbol.Type)); + } + + public static Field Create(Context cx, IFieldSymbol field) => FieldFactory.Instance.CreateEntity(cx, field); + + // Do not populate backing fields. + // Populate Tuple fields. + public override bool NeedsPopulation => + (base.NeedsPopulation && !symbol.IsImplicitlyDeclared) || symbol.ContainingType.IsTupleType; + + public override void Populate() + { + ExtractAttributes(); + ContainingType.ExtractGenerics(); + + Field unboundFieldKey = Field.Create(Context, symbol.OriginalDefinition); + Context.Emit(Tuples.fields(this, (symbol.IsConst ? 2 : 1), symbol.Name, ContainingType, Type.TypeRef, unboundFieldKey)); + + ExtractModifiers(); + + if (symbol.IsVolatile) + Modifier.HasModifier(Context, this, "volatile"); + + if (symbol.IsConst) + { + Modifier.HasModifier(Context, this, "const"); + + if (symbol.HasConstantValue) + { + Context.Emit(Tuples.constant_value(this, Expression.ValueAsString(symbol.ConstantValue))); + } + } + + foreach (var l in Locations) + Context.Emit(Tuples.field_location(this, l)); + + if (!IsSourceDeclaration || !symbol.FromSource()) + return; + + Context.BindComments(this, Location.symbol); + + int child = 0; + foreach (var initializer in + symbol.DeclaringSyntaxReferences. + Select(n => n.GetSyntax()). + OfType(). + Where(n => n.Initializer != null)) + { + Context.PopulateLater(() => + { + Expression.CreateFromNode(new ExpressionNodeInfo(Context, initializer.Initializer.Value, this, child++)); + }); + } + + foreach (var initializer in symbol.DeclaringSyntaxReferences. + Select(n => n.GetSyntax()). + OfType(). + Where(n => n.EqualsValue != null)) + { + // Mark fields that have explicit initializers. + var expr = new Expression(new ExpressionInfo(Context, Type, Context.Create(initializer.EqualsValue.Value.FixedLocation()), Kinds.ExprKind.FIELD_ACCESS, this, child++, false, null)); + Context.Emit(Tuples.expr_access(expr, this)); + } + + if (IsSourceDeclaration) + foreach (var syntax in symbol.DeclaringSyntaxReferences. + Select(d => d.GetSyntax()).OfType(). + Select(d => d.Parent).OfType()) + TypeMention.Create(Context, syntax.Type, this, Type); + } + + readonly Lazy type; + public Type Type => type.Value; + + public override IId Id => new Key(ContainingType, ".", symbol.Name, ";field"); + + bool IExpressionParentEntity.IsTopLevelParent => true; + + class FieldFactory : ICachedEntityFactory + { + public static readonly FieldFactory Instance = new FieldFactory(); + + public Field Create(Context cx, IFieldSymbol init) => new Field(cx, init); + } + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.PushesLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Indexer.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Indexer.cs new file mode 100644 index 00000000000..8fc7e20ccaf --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Indexer.cs @@ -0,0 +1,138 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class Indexer : Property, IExpressionParentEntity + { + protected Indexer(Context cx, IPropertySymbol init) + : base(cx, init) { } + + Indexer OriginalDefinition => IsSourceDeclaration ? this : Create(Context, symbol.OriginalDefinition); + + public override void Populate() + { + var type = Type.Create(Context, symbol.Type); + Context.Emit(Tuples.indexers(this, symbol.GetName(useMetadataName: true), ContainingType, type.TypeRef, OriginalDefinition)); + foreach (var l in Locations) + Context.Emit(Tuples.indexer_location(this, l)); + + var getter = symbol.GetMethod; + var setter = symbol.SetMethod; + + if (getter == null && setter == null) + Context.ModelError(symbol, "No indexer accessor defined"); + + if (getter != null) + { + Getter = Accessor.Create(Context, getter); + } + + if (setter != null) + { + Setter = Accessor.Create(Context, setter); + } + + for (var i = 0; i < symbol.Parameters.Length; ++i) + { + var original = Parameter.Create(Context, symbol.OriginalDefinition.Parameters[i], OriginalDefinition); + Parameter.Create(Context, symbol.Parameters[i], this, original); + } + + if (getter != null) + { + Getter = Accessor.Create(Context, getter); + Context.Emit(Tuples.accessors(Getter, 1, getter.Name, this, Getter.OriginalDefinition)); + + Context.Emit(Tuples.accessor_location(Getter, Getter.Location)); + Getter.Overrides(); + Getter.ExtractModifiers(); + } + + if (setter != null) + { + Setter = Accessor.Create(Context, setter); + Context.Emit(Tuples.accessors(Setter, 2, setter.Name, this, Setter.OriginalDefinition)); + + Context.Emit(Tuples.accessor_location(Setter, Setter.Location)); + Setter.Overrides(); + Setter.ExtractModifiers(); + } + + if (IsSourceDeclaration) + { + var expressionBody = ExpressionBody; + if (expressionBody != null) + { + // The expression may need to reference parameters in the getter. + // So we need to arrange that the expression is populated after the getter. + Context.PopulateLater(() => Expression.CreateFromNode(new ExpressionNodeInfo(Context, expressionBody, this, 0) { Type = Type.Create(Context, symbol.Type) })); + } + } + + ExtractModifiers(); + BindComments(); + + var declSyntaxReferences = IsSourceDeclaration + ? symbol.DeclaringSyntaxReferences. + Select(d => d.GetSyntax()).OfType().ToArray() + : Enumerable.Empty(); + + foreach (var explicitInterface in symbol.ExplicitInterfaceImplementations.Select(impl => Type.Create(Context, impl.ContainingType))) + { + Context.Emit(Tuples.explicitly_implements(this, explicitInterface.TypeRef)); + + foreach (var syntax in declSyntaxReferences) + TypeMention.Create(Context, syntax.ExplicitInterfaceSpecifier.Name, this, explicitInterface); + } + + + foreach (var syntax in declSyntaxReferences) + TypeMention.Create(Context, syntax.Type, this, type); + } + + + public static new Indexer Create(Context cx, IPropertySymbol prop) => IndexerFactory.Instance.CreateEntity(cx, prop); + + public override IId Id + { + get + { + return new Key(tb => + { + tb.Append(ContainingType); + tb.Append("."); + tb.Append(symbol.MetadataName); + tb.Append("("); + tb.BuildList(",", symbol.Parameters, (p, tb0) => tb0.Append(Type.Create(Context, p.Type))); + tb.Append(");indexer"); + }); + } + } + + public override Microsoft.CodeAnalysis.Location FullLocation + { + get + { + return + symbol. + DeclaringSyntaxReferences. + Select(r => r.GetSyntax()). + OfType(). + Select(s => s.GetLocation()). + Concat(symbol.Locations). + First(); + } + } + + bool IExpressionParentEntity.IsTopLevelParent => true; + + class IndexerFactory : ICachedEntityFactory + { + public static readonly IndexerFactory Instance = new IndexerFactory(); + + public Indexer Create(Context cx, IPropertySymbol init) => new Indexer(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalFunction.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalFunction.cs new file mode 100644 index 00000000000..7d808e0fd3a --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalFunction.cs @@ -0,0 +1,48 @@ +using Microsoft.CodeAnalysis; + +namespace Semmle.Extraction.CSharp.Entities +{ + class LocalFunction : Method + { + LocalFunction(Context cx, IMethodSymbol init) : base(cx, init) + { + } + + public override IId Id + { + get + { + return new Key(tb => + { + tb.Append(Location); + if (symbol.IsGenericMethod && !IsSourceDeclaration) + { + tb.Append("<"); + tb.BuildList(",", symbol.TypeArguments, (ta, tb0) => AddSignatureTypeToId(Context, tb0, symbol, ta)); + tb.Append(">"); + } + + tb.Append(";localfunction"); + }); + } + } + + public static new LocalFunction Create(Context cx, IMethodSymbol field) => LocalFunctionFactory.Instance.CreateEntity(cx, field); + + class LocalFunctionFactory : ICachedEntityFactory + { + public static readonly LocalFunctionFactory Instance = new LocalFunctionFactory(); + + public LocalFunction Create(Context cx, IMethodSymbol init) => new LocalFunction(cx, init); + } + + public override void Populate() + { + PopulateMethod(); + var originalDefinition = IsSourceDeclaration ? this : Create(Context, symbol.OriginalDefinition); + var returnType = Type.Create(Context, symbol.ReturnType); + Context.Emit(Tuples.local_functions(this, symbol.Name, returnType, originalDefinition)); + ExtractRefReturn(); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalVariable.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalVariable.cs new file mode 100644 index 00000000000..163f81040a5 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalVariable.cs @@ -0,0 +1,95 @@ +using System; +using Microsoft.CodeAnalysis; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities +{ + class LocalVariable : CachedSymbol + { + LocalVariable(Context cx, ISymbol init, Expression parent, bool isVar, Extraction.Entities.Location declLocation) + : base(cx, init) + { + Parent = parent; + IsVar = isVar; + DeclLocation = declLocation; + } + + readonly Expression Parent; + readonly bool IsVar; + readonly Extraction.Entities.Location DeclLocation; + + public override IId Id => new Key(Parent, "_", symbol.Name, ";localvar"); + + public override void Populate() + { + Context.Emit(Tuples.localvars( + this, + IsRef ? 3 : IsConst ? 2 : 1, + symbol.Name, + IsVar ? 1 : 0, + Type.TypeRef, + Parent)); + + Context.Emit(Tuples.localvar_location(this, DeclLocation)); + + DefineConstantValue(); + } + + public static LocalVariable Create(Context cx, ISymbol local, Expression parent, bool isVar, Extraction.Entities.Location declLocation) + { + return LocalVariableFactory.Instance.CreateEntity(cx, local, parent, isVar, declLocation); + } + + /// + /// Gets the local variable entity for which must + /// already have been created. + /// + public static LocalVariable GetAlreadyCreated(Context cx, ISymbol local) => LocalVariableFactory.Instance.CreateEntity(cx, local, null, false, null); + + bool IsConst + { + get + { + var local = symbol as ILocalSymbol; + return local != null && local.IsConst; + } + } + + bool IsRef + { + get + { + var local = symbol as ILocalSymbol; + return local != null && local.IsRef; + } + } + + Type Type + { + get + { + var local = symbol as ILocalSymbol; + return local == null ? Parent.Type : Type.Create(Context, local.Type); + } + } + + void DefineConstantValue() + { + var local = symbol as ILocalSymbol; + if (local != null && local.HasConstantValue) + { + Context.Emit(Tuples.constant_value(this, Expression.ValueAsString(local.ConstantValue))); + } + } + + class LocalVariableFactory : ICachedEntityFactory<(ISymbol, Expression, bool, Extraction.Entities.Location), LocalVariable> + { + public static readonly LocalVariableFactory Instance = new LocalVariableFactory(); + + public LocalVariable Create(Context cx, (ISymbol, Expression, bool, Extraction.Entities.Location) init) => + new LocalVariable(cx, init.Item1, init.Item2, init.Item3, init.Item4); + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NeedsLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Method.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Method.cs new file mode 100644 index 00000000000..513cb52565d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Method.cs @@ -0,0 +1,368 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using System.Collections.Generic; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + public abstract class Method : CachedSymbol, IExpressionParentEntity, IStatementParentEntity + { + public Method(Context cx, IMethodSymbol init) + : base(cx, init) { } + + protected void ExtractParameters() + { + var originalMethod = OriginalDefinition; + IEnumerable parameters = symbol.Parameters; + IEnumerable originalParameters = originalMethod.symbol.Parameters; + + if (IsReducedExtension) + { + if (this == originalMethod) + { + // Non-generic reduced extensions must be extracted exactly like the + // non-reduced counterparts + parameters = symbol.ReducedFrom.Parameters; + } + else + { + // Constructed reduced extensions are special because their non-reduced + // counterparts are not constructed. Therefore, we need to manually add + // the `this` parameter based on the type of the receiver + var originalThisParamSymbol = originalMethod.symbol.Parameters.First(); + var originalThisParam = Parameter.Create(Context, originalThisParamSymbol, originalMethod); + ConstructedExtensionParameter.Create(Context, this, originalThisParam); + originalParameters = originalParameters.Skip(1); + } + } + + foreach (var p in parameters.Zip(originalParameters, (paramSymbol, originalParam) => new { paramSymbol, originalParam })) + { + var original = Equals(p.paramSymbol, p.originalParam) ? null : Parameter.Create(Context, p.originalParam, originalMethod); + Parameter.Create(Context, p.paramSymbol, this, original); + } + + if (symbol.IsVararg) + { + // Mono decided that "__arglist" should be an explicit parameter, + // so now we need to populate it. + + VarargsParam.Create(Context, this); + } + } + + /// + /// Extracts constructor initializers. + /// + protected virtual void ExtractInitializers() + { + // Normal methods don't have initializers, + // so there's nothing to extract. + } + + void ExtractMethodBody() + { + if (!IsSourceDeclaration) + return; + + var block = Block; + var expr = ExpressionBody; + + if (block != null || expr != null) + Context.PopulateLater( + () => + { + ExtractInitializers(); + if (block != null) + Statements.Block.Create(Context, block, this, 0); + else + Expression.Create(Context, expr, this, 0); + + Context.NumberOfLines(symbol, this); + }); + } + + public void Overrides() + { + foreach (var explicitInterface in symbol.ExplicitInterfaceImplementations. + Where(sym => sym.MethodKind == MethodKind.Ordinary). + Select(impl => Type.Create(Context, impl.ContainingType))) + { + Context.Emit(Tuples.explicitly_implements(this, explicitInterface.TypeRef)); + + if (IsSourceDeclaration) + foreach (var syntax in symbol.DeclaringSyntaxReferences.Select(d => d.GetSyntax()).OfType()) + TypeMention.Create(Context, syntax.ExplicitInterfaceSpecifier.Name, this, explicitInterface); + } + + if (symbol.OverriddenMethod != null) + { + Context.Emit(Tuples.overrides(this, Method.Create(Context, symbol.OverriddenMethod))); + } + } + + /// + /// Factored out to share logic between `Method` and `UserOperator`. + /// + protected static void BuildMethodId(Method m, ITrapBuilder tb) + { + tb.Append(m.ContainingType); + + AddExplicitInterfaceQualifierToId(m.Context, tb, m.symbol.ExplicitInterfaceImplementations); + + tb.Append(".").Append(m.symbol.Name); + + if (m.symbol.IsGenericMethod) + { + if (Equals(m.symbol, m.symbol.OriginalDefinition)) + { + tb.Append("`").Append(m.symbol.TypeParameters.Length); + } + else + { + tb.Append("<"); + tb.BuildList(",", m.symbol.TypeArguments, (ta, tb0) => AddSignatureTypeToId(m.Context, tb0, m.symbol, ta)); + tb.Append(">"); + } + } + + AddParametersToId(m.Context, tb, m.symbol); + switch (m.symbol.MethodKind) + { + case MethodKind.PropertyGet: + tb.Append(";getter"); + break; + case MethodKind.PropertySet: + tb.Append(";setter"); + break; + case MethodKind.EventAdd: + tb.Append(";adder"); + break; + case MethodKind.EventRaise: + tb.Append(";raiser"); + break; + case MethodKind.EventRemove: + tb.Append(";remover"); + break; + default: + tb.Append(";method"); + break; + } + } + + public override IId Id => new Key(tb => BuildMethodId(this, tb)); + + /// + /// Adds an appropriate label ID to the trap builder + /// for the type belonging to the signature of method + /// . + /// + /// For methods without type parameters this will always add the key of the + /// corresponding type. + /// + /// For methods with type parameters, this will add the key of the + /// corresponding type if the type does *not* contain one of the method + /// type parameters, otherwise it will add a textual representation of + /// the type. This distinction is required because type parameter IDs + /// refer to their declaring methods. + /// + /// Example: + /// + /// + /// int Count<T>(IEnumerable items) + /// + /// + /// The label definitions for Count (#4) and T + /// (#5) will look like: + /// + /// + /// #1=<label for System.Int32> + /// #2=<label for type containing Count> + /// #3=<label for IEnumerable`1> + /// #4=@"{#1} {#2}.Count`2(#3);method" + /// #5=@"{#4}T;typeparameter" + /// + /// + /// Note how int is referenced in the label definition #3 for + /// Count, while T[] is represented textually in order + /// to make the reference to #3 in the label definition #4 for + /// T valid. + /// + protected static void AddSignatureTypeToId(Context cx, ITrapBuilder tb, IMethodSymbol method, ITypeSymbol type) + { + if (type.ContainsTypeParameters(cx, method)) + type.BuildTypeId(cx, tb, (cx0, tb0, type0) => AddSignatureTypeToId(cx, tb0, method, type0)); + else + tb.Append(Type.Create(cx, type)); + } + + protected static void AddParametersToId(Context cx, ITrapBuilder tb, IMethodSymbol method) + { + tb.Append("("); + tb.AppendList(",", AddParameterPartsToId(cx, tb, method)); + tb.Append(")"); + } + + // This is a slight abuse of ITrapBuilder.AppendList(). + // yield return "" is used to insert a list separator + // at the desired location. + static IEnumerable AddParameterPartsToId(Context cx, ITrapBuilder tb, IMethodSymbol method) + { + if (method.MethodKind == MethodKind.ReducedExtension) + { + AddSignatureTypeToId(cx, tb, method, method.ReceiverType); + yield return ""; // The next yield return outputs a "," + } + + foreach (var param in method.Parameters) + { + yield return ""; // Maybe print "," + AddSignatureTypeToId(cx, tb, method, param.Type); + switch (param.RefKind) + { + case RefKind.Out: + yield return " out"; + break; + case RefKind.Ref: + yield return " ref"; + break; + } + } + + if (method.IsVararg) + { + yield return "__arglist"; + } + } + + public static void AddExplicitInterfaceQualifierToId(Context cx, ITrapBuilder tb, IEnumerable explicitInterfaceImplementations) + { + if (explicitInterfaceImplementations.Any()) + { + tb.AppendList(",", explicitInterfaceImplementations.Select(impl => cx.CreateEntity(impl.ContainingType))); + } + } + + public virtual string Name => symbol.Name; + + /// + /// Creates a method of the appropriate subtype. + /// + /// + /// + /// + public static Method Create(Context cx, IMethodSymbol methodDecl) + { + if (methodDecl == null) return null; + + switch (methodDecl.MethodKind) + { + case MethodKind.StaticConstructor: + case MethodKind.Constructor: + return Constructor.Create(cx, methodDecl); + case MethodKind.ReducedExtension: + case MethodKind.Ordinary: + case MethodKind.DelegateInvoke: + case MethodKind.ExplicitInterfaceImplementation: + return OrdinaryMethod.Create(cx, methodDecl); + case MethodKind.Destructor: + return Destructor.Create(cx, methodDecl); + case MethodKind.PropertyGet: + case MethodKind.PropertySet: + return Accessor.Create(cx, methodDecl); + case MethodKind.EventAdd: + case MethodKind.EventRemove: + return EventAccessor.Create(cx, methodDecl); + case MethodKind.UserDefinedOperator: + case MethodKind.BuiltinOperator: + return UserOperator.Create(cx, methodDecl); + case MethodKind.Conversion: + return Conversion.Create(cx, methodDecl); + case MethodKind.AnonymousFunction: + throw new InternalError(methodDecl, "Attempt to create a lambda"); + case MethodKind.LocalFunction: + return LocalFunction.Create(cx, methodDecl); + default: + throw new InternalError(methodDecl, "Unhandled method '{0}' of kind '{1}'", methodDecl, methodDecl.MethodKind); + } + } + + public Method OriginalDefinition => + IsReducedExtension + ? Create(Context, symbol.ReducedFrom) + : Create(Context, symbol.OriginalDefinition); + + public override Microsoft.CodeAnalysis.Location FullLocation => ReportingLocation; + + public override bool IsSourceDeclaration => symbol.IsSourceDeclaration(); + + /// + /// Whether this method has type parameters. + /// + public bool IsGeneric => symbol.IsGenericMethod; + + /// + /// Whether this method has unbound type parameters. + /// + public bool IsUnboundGeneric => IsGeneric && Equals(symbol.ConstructedFrom, symbol); + + public bool IsBoundGeneric => IsGeneric && !IsUnboundGeneric; + + bool IsReducedExtension => symbol.MethodKind == MethodKind.ReducedExtension; + + protected IMethodSymbol ConstructedFromSymbol => symbol.ConstructedFrom.ReducedFrom ?? symbol.ConstructedFrom; + + bool IExpressionParentEntity.IsTopLevelParent => true; + + bool IStatementParentEntity.IsTopLevelParent => true; + + protected void ExtractGenerics() + { + var isFullyConstructed = IsBoundGeneric; + + if (IsGeneric) + { + int child = 0; + + if (isFullyConstructed) + { + Context.Emit(Tuples.is_constructed(this)); + Context.Emit(Tuples.constructed_generic(this, Method.Create(Context, ConstructedFromSymbol))); + foreach (var tp in symbol.TypeArguments) + { + Context.Emit(Tuples.type_arguments(Type.Create(Context, tp), child++, this)); + } + } + else + { + Context.Emit(Tuples.is_generic(this)); + foreach (var typeParam in symbol.TypeParameters.Select(tp => TypeParameter.Create(Context, tp))) + { + Context.Emit(Tuples.type_parameters(typeParam, child++, this)); + } + } + } + } + + protected void ExtractRefReturn() + { + if (symbol.ReturnsByRef) + Context.Emit(Tuples.ref_returns(this)); + if (symbol.ReturnsByRefReadonly) + Context.Emit(Tuples.ref_readonly_returns(this)); + } + + protected void PopulateMethod() + { + // Common population code for all callables + BindComments(); + ExtractAttributes(); + ExtractParameters(); + ExtractMethodBody(); + ExtractGenerics(); + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.PushesLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs new file mode 100644 index 00000000000..bca30cea96d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs @@ -0,0 +1,169 @@ +using Microsoft.CodeAnalysis; +using System; +using System.Reflection; + +namespace Semmle.Extraction.CSharp.Entities +{ + /// + /// Provide a "Key" object to allow modifiers to exist as entities in the extractor + /// hash map. (Raw strings would work as keys but might clash with other types). + /// + class ModifierKey : Object + { + public readonly string name; + + public ModifierKey(string m) + { + name = m; + } + + public override bool Equals(Object obj) + { + return obj.GetType() == GetType() && name == ((ModifierKey)obj).name; + } + + public override int GetHashCode() => 13 * name.GetHashCode(); + } + + class Modifier : Extraction.CachedEntity + { + Modifier(Context cx, ModifierKey init) + : base(cx, init) { } + + public override Microsoft.CodeAnalysis.Location ReportingLocation => null; + + public override IId Id => new Key(symbol.name, ";modifier"); + + public override bool NeedsPopulation => true; + + public override void Populate() + { + Context.Emit(new Tuple("modifiers", Label, symbol.name)); + } + + public static string AccessbilityModifier(Accessibility access) + { + switch (access) + { + case Accessibility.Private: + return "private"; + case Accessibility.Protected: + return "protected"; + case Accessibility.Public: + return "public"; + case Accessibility.Internal: + return "internal"; + default: + throw new InternalError("Unavailable modifier combination"); + } + } + + public static void HasAccessibility(Context cx, IEntity type, Accessibility access) + { + switch (access) + { + case Accessibility.Private: + case Accessibility.Public: + case Accessibility.Protected: + case Accessibility.Internal: + HasModifier(cx, type, Modifier.AccessbilityModifier(access)); + break; + case Accessibility.NotApplicable: + break; + case Accessibility.ProtectedOrInternal: + HasModifier(cx, type, "protected"); + HasModifier(cx, type, "internal"); + break; + case Accessibility.ProtectedAndInternal: + HasModifier(cx, type, "protected"); + HasModifier(cx, type, "private"); + break; + default: + throw new InternalError("Unhandled Microsoft.CodeAnalysis.Accessibility value: {0}", access); + } + } + + public static void HasModifier(Context cx, IEntity target, string modifier) + { + cx.Emit(Tuples.has_modifiers(target, Modifier.Create(cx, modifier))); + } + + public static void ExtractModifiers(Context cx, IEntity key, ISymbol symbol) + { + bool interfaceDefinition = symbol.ContainingType != null + && symbol.ContainingType.Kind == SymbolKind.NamedType + && symbol.ContainingType.TypeKind == TypeKind.Interface; + + Modifier.HasAccessibility(cx, key, symbol.DeclaredAccessibility); + if (symbol.Kind == SymbolKind.ErrorType) + cx.Emit(Tuples.has_modifiers(key, Modifier.Create(cx, Accessibility.Public))); + + if (symbol.IsAbstract && (symbol.Kind != SymbolKind.NamedType || ((INamedTypeSymbol)symbol).TypeKind != TypeKind.Interface) && !interfaceDefinition) + Modifier.HasModifier(cx, key, "abstract"); + + if (symbol.IsSealed) + HasModifier(cx, key, "sealed"); + + bool fromSource = symbol.DeclaringSyntaxReferences.Length > 0; + + if (symbol.IsStatic && !(symbol.Kind == SymbolKind.Field && ((IFieldSymbol)symbol).IsConst && !fromSource)) + HasModifier(cx, key, "static"); + + if (symbol.IsVirtual) + HasModifier(cx, key, "virtual"); + + // For some reason, method in interfaces are "virtual", not "abstract" + if (symbol.IsAbstract && interfaceDefinition) + HasModifier(cx, key, "virtual"); + + if (symbol.Kind == SymbolKind.Field && ((IFieldSymbol)symbol).IsReadOnly) + HasModifier(cx, key, "readonly"); + + if (symbol.IsOverride) + HasModifier(cx, key, "override"); + + if (symbol.Kind == SymbolKind.Method && ((IMethodSymbol)symbol).IsAsync) + HasModifier(cx, key, "async"); + + if (symbol.IsExtern) + HasModifier(cx, key, "extern"); + + foreach (var modifier in symbol.GetSourceLevelModifiers()) + HasModifier(cx, key, modifier); + + if (symbol.Kind == SymbolKind.NamedType) + { + INamedTypeSymbol nt = symbol as INamedTypeSymbol; + if (nt.TypeKind == TypeKind.Struct) + { + // Sadly, these properties are internal so cannot be accessed directly. + // This seems to be a deficiency in the model. + var readonlyProperty = nt.GetType().GetProperty("IsReadOnly", BindingFlags.NonPublic | BindingFlags.Instance); + var isByRefProperty = nt.GetType().GetProperty("IsByRefLikeType", BindingFlags.NonPublic | BindingFlags.Instance); + + bool isReadOnly = (bool)readonlyProperty.GetValue(nt); + bool isByRefLikeType = (bool)isByRefProperty.GetValue(nt); + + if (isReadOnly) + HasModifier(cx, key, "readonly"); + if (isByRefLikeType) + HasModifier(cx, key, "ref"); + } + } + } + + public static Modifier Create(Context cx, string modifier) => + ModifierFactory.Instance.CreateEntity(cx, new ModifierKey(modifier)); + + public static Modifier Create(Context cx, Accessibility access) => + ModifierFactory.Instance.CreateEntity(cx, new ModifierKey(AccessbilityModifier(access))); + + class ModifierFactory : ICachedEntityFactory + { + public static readonly ModifierFactory Instance = new ModifierFactory(); + + public Modifier Create(Context cx, ModifierKey init) => new Modifier(cx, init); + } + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.OptionalLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Namespace.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Namespace.cs new file mode 100644 index 00000000000..f66ac8e6d60 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Namespace.cs @@ -0,0 +1,45 @@ +using Microsoft.CodeAnalysis; + +namespace Semmle.Extraction.CSharp.Entities +{ + class Namespace : CachedEntity + { + Namespace(Context cx, INamespaceSymbol init) + : base(cx, init) { } + + public override Microsoft.CodeAnalysis.Location ReportingLocation => null; + + public override void Populate() + { + Context.Emit(Tuples.namespaces(this, symbol.Name)); + + if (symbol.ContainingNamespace != null) + { + Namespace parent = Create(Context, symbol.ContainingNamespace); + Context.Emit(Tuples.parent_namespace(this, parent)); + } + } + + public override bool NeedsPopulation => true; + + public override IId Id + { + get + { + return symbol.IsGlobalNamespace ? new Key(";namespace") : + new Key(Create(Context, symbol.ContainingNamespace), ".", symbol.Name, ";namespace"); + } + } + + public static Namespace Create(Context cx, INamespaceSymbol ns) => NamespaceFactory.Instance.CreateEntity(cx, ns); + + class NamespaceFactory : ICachedEntityFactory + { + public static readonly NamespaceFactory Instance = new NamespaceFactory(); + + public Namespace Create(Context cx, INamespaceSymbol init) => new Namespace(cx, init); + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs new file mode 100644 index 00000000000..d889361649d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs @@ -0,0 +1,41 @@ + +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Entities; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class NamespaceDeclaration : FreshEntity + { + public NamespaceDeclaration(Context cx, NamespaceDeclarationSyntax node, NamespaceDeclaration parent) + : base(cx) + { + var ns = Namespace.Create(cx, (INamespaceSymbol)cx.Model(node).GetSymbolInfo(node.Name).Symbol); + cx.Emit(Tuples.namespace_declarations(this, ns)); + cx.Emit(Tuples.namespace_declaration_location(this, cx.Create(node.Name.GetLocation()))); + + var visitor = new Populators.TypeOrNamespaceVisitor(cx, this); + + foreach (var member in node.Members.Cast().Concat(node.Usings)) + { + member.Accept(visitor); + } + + if (parent != null) + { + cx.Emit(Tuples.parent_namespace_declaration(this, parent)); + } + } + + public void Extract(Context cx, NamespaceDeclarationSyntax decl) + { + decl.Accept(new Populators.TypeOrNamespaceVisitor(cx, this)); + } + + public static NamespaceDeclaration Create(Context cx, NamespaceDeclarationSyntax decl, NamespaceDeclaration parent) => new NamespaceDeclaration(cx, decl, parent); + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs new file mode 100644 index 00000000000..88895025446 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs @@ -0,0 +1,60 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class OrdinaryMethod : Method + { + OrdinaryMethod(Context cx, IMethodSymbol init) + : base(cx, init) { } + + public override string Name => symbol.GetName(); + + public IMethodSymbol SourceDeclaration + { + get + { + var reducedFrom = symbol.ReducedFrom ?? symbol; + return reducedFrom.OriginalDefinition; + } + } + + public override Microsoft.CodeAnalysis.Location ReportingLocation => symbol.GetSymbolLocation(); + + public override void Populate() + { + PopulateMethod(); + ExtractModifiers(); + ContainingType.ExtractGenerics(); + + var returnType = Type.Create(Context, symbol.ReturnType); + Context.Emit(Tuples.methods(this, Name, ContainingType, returnType.TypeRef, OriginalDefinition)); + + if (IsSourceDeclaration) + foreach (var declaration in symbol.DeclaringSyntaxReferences.Select(s => s.GetSyntax()).OfType()) + { + Context.BindComments(this, declaration.Identifier.GetLocation()); + TypeMention.Create(Context, declaration.ReturnType, this, returnType); + } + + foreach (var l in Locations) + Context.Emit(Tuples.method_location(this, l)); + + ExtractGenerics(); + Overrides(); + ExtractRefReturn(); + ExtractCompilerGenerated(); + } + + public new static OrdinaryMethod Create(Context cx, IMethodSymbol method) => OrdinaryMethodFactory.Instance.CreateEntity(cx, method); + + class OrdinaryMethodFactory : ICachedEntityFactory + { + public static readonly OrdinaryMethodFactory Instance = new OrdinaryMethodFactory(); + + public OrdinaryMethod Create(Context cx, IMethodSymbol init) => new OrdinaryMethod(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs new file mode 100644 index 00000000000..b5858d5c5e9 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs @@ -0,0 +1,290 @@ +using System; +using Microsoft.CodeAnalysis; +using Semmle.Extraction.CSharp.Populators; +using System.Linq; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities +{ + public class Parameter : CachedSymbol, IExpressionParentEntity + { + protected IEntity Parent; + protected readonly Parameter Original; + + protected Parameter(Context cx, IParameterSymbol init, IEntity parent, Parameter original) + : base(cx, init) + { + Parent = parent; + Original = original ?? this; + } + + public override Microsoft.CodeAnalysis.Location ReportingLocation => symbol.GetSymbolLocation(); + + public enum Kind + { + None, Ref, Out, Params, This, In + } + + protected virtual int Ordinal + { + get + { + // For some reason, methods of kind ReducedExtension + // omit the "this" parameter, so the parameters are + // actually numbered from 1. + // This is to be consistent from the original (unreduced) extension method. + var method = symbol.ContainingSymbol as IMethodSymbol; + bool isReducedExtension = method != null && method.MethodKind == MethodKind.ReducedExtension; + return symbol.Ordinal + (isReducedExtension ? 1 : 0); + } + } + + Kind ParamKind + { + get + { + switch (symbol.RefKind) + { + case RefKind.Out: + return Kind.Out; + case RefKind.Ref: + return Kind.Ref; + case RefKind.In: + return Kind.In; + default: + if (symbol.IsParams) return Kind.Params; + + if (Ordinal == 0) + { + var method = symbol.ContainingSymbol as IMethodSymbol; + if (method != null && method.IsExtensionMethod) return Kind.This; + } + return Kind.None; + } + } + } + + public static Parameter Create(Context cx, IParameterSymbol param, IEntity parent, Parameter original = null) => + ParameterFactory.Instance.CreateEntity(cx, param, parent, original); + + /// + /// Gets the parameter entity for which must + /// already have been created. + /// + public static Parameter GetAlreadyCreated(Context cx, IParameterSymbol param) => + ParameterFactory.Instance.CreateEntity(cx, param, null, null); + + public override IId Id + { + get + { + // This is due to a bug in Roslyn when ValueTuple.cs is extracted. + // The parameter symbols don't match up properly so we don't have a parent. + if (Parent == null) + Parent = Method.Create(Context, symbol.ContainingSymbol as IMethodSymbol); + return new Key(Parent, "_", Ordinal, ";parameter"); + } + } + + public override bool NeedsPopulation => true; + + string Name + { + get + { + // Very rarely, two parameters have the same name according to the data model. + // This breaks our database constraints. + // Generate an impossible name to ensure that it doesn't conflict. + int conflictingCount = symbol.ContainingSymbol.GetParameters().Count(p => p.Ordinal < symbol.Ordinal && p.Name == symbol.Name); + return conflictingCount > 0 ? symbol.Name + "`" + conflictingCount : symbol.Name; + } + } + + public override void Populate() + { + ExtractAttributes(); + + if (symbol.Name != Original.symbol.Name) + Context.ModelError(symbol, "Inconsistent parameter declaration"); + + var type = Type.Create(Context, symbol.Type); + Context.Emit(Tuples.@params(this, Name, type.TypeRef, Ordinal, ParamKind, Parent, Original)); + + foreach (var l in symbol.Locations) + Context.Emit(Tuples.param_location(this, Context.Create(l))); + + if (!IsSourceDeclaration || !symbol.FromSource()) + return; + + BindComments(); + + if (IsSourceDeclaration) + foreach (var syntax in symbol.DeclaringSyntaxReferences. + Select(d => d.GetSyntax()). + OfType(). + Where(s => s.Type != null)) + TypeMention.Create(Context, syntax.Type, this, type); + + if (symbol.HasExplicitDefaultValue && Context.Defines(symbol)) + { + // This is a slight bug in the dbscheme + // We should really define param_default(param, string) + // And use parameter child #0 to encode the default expression. + var defaultValue = GetParameterDefaultValue(symbol); + if (defaultValue == null) + { + // In case this parameter belongs to an accessor of an indexer, we need + // to get the default value from the corresponding parameter belonging + // to the indexer itself + var method = (IMethodSymbol)symbol.ContainingSymbol; + if (method != null) + { + var i = method.Parameters.IndexOf(symbol); + var indexer = (IPropertySymbol)method.AssociatedSymbol; + if (indexer != null) + defaultValue = GetParameterDefaultValue(indexer.Parameters[i]); + } + } + + if (defaultValue != null) + { + Context.PopulateLater(() => + { + Expression.Create(Context, defaultValue.Value, this, 0); + }); + } + } + } + + public override bool IsSourceDeclaration => symbol.IsSourceDeclaration(); + + bool IExpressionParentEntity.IsTopLevelParent => true; + + static EqualsValueClauseSyntax GetParameterDefaultValue(IParameterSymbol parameter) + { + var syntax = parameter.DeclaringSyntaxReferences.Select(@ref => @ref.GetSyntax()).OfType().FirstOrDefault(); + return syntax != null ? syntax.Default : null; + } + + class ParameterFactory : ICachedEntityFactory<(IParameterSymbol, IEntity, Parameter), Parameter> + { + public static readonly ParameterFactory Instance = new ParameterFactory(); + + public Parameter Create(Context cx, (IParameterSymbol, IEntity, Parameter) init) => new Parameter(cx, init.Item1, init.Item2, init.Item3); + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.OptionalLabel; + } + + class VarargsType : Type + { + VarargsType(Context cx) + : base(cx, null) { } + + public override void Populate() + { + Context.Emit(Tuples.types(this, Kinds.TypeKind.ARGLIST, "__arglist")); + Context.Emit(Tuples.parent_namespace(this, Namespace.Create(Context, Context.Compilation.GlobalNamespace))); + Modifier.HasModifier(Context, this, "public"); + } + + public override bool NeedsPopulation => true; + + public sealed override IId Id => new Key("__arglist;type"); + + public override int GetHashCode() + { + return 98735267; + } + + public override bool Equals(object obj) + { + return obj != null && obj.GetType() == typeof(VarargsType); + } + + public static VarargsType Create(Context cx) => VarargsTypeFactory.Instance.CreateEntity(cx, null); + + class VarargsTypeFactory : ICachedEntityFactory + { + public static readonly VarargsTypeFactory Instance = new VarargsTypeFactory(); + + public VarargsType Create(Context cx, string init) => new VarargsType(cx); + } + } + + class VarargsParam : Parameter + { + VarargsParam(Context cx, Method methodKey) + : base(cx, null, methodKey, null) { } + + public override void Populate() + { + var typeKey = VarargsType.Create(Context); + // !! Maybe originaldefinition is wrong + Context.Emit(Tuples.@params(this, "", typeKey, Ordinal, Kind.None, Parent, this)); + Context.Emit(Tuples.param_location(this, GeneratedLocation.Create(Context))); + } + + protected override int Ordinal => ((Method)Parent).OriginalDefinition.symbol.Parameters.Length; + + public override int GetHashCode() + { + return 9873567; + } + + public override bool Equals(object obj) + { + return obj != null && obj.GetType() == typeof(VarargsParam); + } + + public static VarargsParam Create(Context cx, Method method) => VarargsParamFactory.Instance.CreateEntity(cx, method); + + class VarargsParamFactory : ICachedEntityFactory + { + public static readonly VarargsParamFactory Instance = new VarargsParamFactory(); + + public VarargsParam Create(Context cx, Method init) => new VarargsParam(cx, init); + } + } + + class ConstructedExtensionParameter : Parameter + { + readonly ITypeSymbol ConstructedType; + + ConstructedExtensionParameter(Context cx, Method method, Parameter original) + : base(cx, original.symbol, method, original) + { + ConstructedType = method.symbol.ReceiverType; + } + + public override void Populate() + { + var typeKey = Type.Create(Context, ConstructedType); + Context.Emit(Tuples.@params(this, Original.symbol.Name, typeKey.TypeRef, 0, Kind.This, Parent, Original)); + Context.Emit(Tuples.param_location(this, Original.Location)); + } + + public override int GetHashCode() => symbol.GetHashCode() + 31 * ConstructedType.GetHashCode(); + + public override bool Equals(object obj) + { + var other = obj as ConstructedExtensionParameter; + if (other == null || other.GetType() != typeof(ConstructedExtensionParameter)) + return false; + + return Equals(symbol, other.symbol) && Equals(ConstructedType, other.ConstructedType); + } + + public static ConstructedExtensionParameter Create(Context cx, Method method, Parameter parameter) => + ExtensionParamFactory.Instance.CreateEntity(cx, (method, parameter)); + + class ExtensionParamFactory : ICachedEntityFactory<(Method, Parameter), ConstructedExtensionParameter> + { + public static readonly ExtensionParamFactory Instance = new ExtensionParamFactory(); + + public ConstructedExtensionParameter Create(Context cx, (Method, Parameter) init) => + new ConstructedExtensionParameter(cx, init.Item1, init.Item2); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs new file mode 100644 index 00000000000..7b26e21be45 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs @@ -0,0 +1,119 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class Property : CachedSymbol, IExpressionParentEntity + { + protected Property(Context cx, IPropertySymbol init) + : base(cx, init) { } + + public override IId Id + { + get + { + return new Key(tb => + { + tb.Append(ContainingType); + tb.Append("."); + Method.AddExplicitInterfaceQualifierToId(Context, tb, symbol.ExplicitInterfaceImplementations); + tb.Append(symbol.Name); + tb.Append(";property"); + }); + } + } + + protected Accessor Getter { get; set; } + protected Accessor Setter { get; set; } + + public override void Populate() + { + ExtractAttributes(); + ExtractModifiers(); + BindComments(); + + var type = Type.Create(Context, symbol.Type); + Context.Emit(Tuples.properties(this, symbol.GetName(), ContainingType, type.TypeRef, Create(Context, symbol.OriginalDefinition))); + + var getter = symbol.GetMethod; + if (getter != null) + Getter = Accessor.Create(Context, getter); + + var setter = symbol.SetMethod; + if (setter != null) + Setter = Accessor.Create(Context, setter); + + var declSyntaxReferences = IsSourceDeclaration ? + symbol.DeclaringSyntaxReferences. + Select(d => d.GetSyntax()).OfType().ToArray() + : Enumerable.Empty(); + + foreach (var explicitInterface in symbol.ExplicitInterfaceImplementations.Select(impl => Type.Create(Context, impl.ContainingType))) + { + Context.Emit(Tuples.explicitly_implements(this, explicitInterface.TypeRef)); + + foreach (var syntax in declSyntaxReferences) + TypeMention.Create(Context, syntax.ExplicitInterfaceSpecifier.Name, this, explicitInterface); + } + + foreach (var l in Locations) + Context.Emit(Tuples.property_location(this, l)); + + if (IsSourceDeclaration && symbol.FromSource()) + { + var expressionBody = ExpressionBody; + if (expressionBody != null) + { + Context.PopulateLater(() => Expression.Create(Context, expressionBody, this, 0)); + } + + foreach (var initializer in declSyntaxReferences. + Select(n => n.Initializer). + Where(i => i != null). + Select(i => i.Value)) + { + Context.PopulateLater(() => Expression.Create(Context, initializer, this, 1)); + } + + foreach (var syntax in declSyntaxReferences) + TypeMention.Create(Context, syntax.Type, this, type); + } + } + + public override Microsoft.CodeAnalysis.Location FullLocation + { + get + { + return + symbol. + DeclaringSyntaxReferences. + Select(r => r.GetSyntax()). + OfType(). + Select(s => s.GetLocation()). + Concat(symbol.Locations). + First(); + } + } + + bool IExpressionParentEntity.IsTopLevelParent => true; + + public static Property Create(Context cx, IPropertySymbol prop) + { + return prop.IsIndexer ? Indexer.Create(cx, prop) : PropertyFactory.Instance.CreateEntity(cx, prop); + } + + public void VisitDeclaration(Context cx, PropertyDeclarationSyntax p) + { + } + + class PropertyFactory : ICachedEntityFactory + { + public static readonly PropertyFactory Instance = new PropertyFactory(); + + public Property Create(Context cx, IPropertySymbol init) => new Property(cx, init); + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statement.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statement.cs new file mode 100644 index 00000000000..8fb755c1805 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statement.cs @@ -0,0 +1,79 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Microsoft.CodeAnalysis.CSharp; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities +{ + /// + /// Whether this entity is the parent of a top-level statement. + /// + public interface IStatementParentEntity : IEntity + { + bool IsTopLevelParent { get; } + } + + abstract class Statement : FreshEntity, IExpressionParentEntity, IStatementParentEntity + { + protected Statement(Context cx) : base(cx) { } + + public static Statement Create(Context cx, StatementSyntax node, Statement parent, int child) => + Statements.Factory.Create(cx, node, parent, child); + + /// + /// How many statements does this take up in a block. + /// The default is 1, however labelled statements can be more. + /// + public virtual int NumberOfStatements => 1; + + public override Microsoft.CodeAnalysis.Location ReportingLocation => GetStatementSyntax().GetLocation(); + + bool IExpressionParentEntity.IsTopLevelParent => false; + + bool IStatementParentEntity.IsTopLevelParent => false; + + protected abstract CSharpSyntaxNode GetStatementSyntax(); + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NeedsLabel; + } + + abstract class Statement : Statement where TSyntax : CSharpSyntaxNode + { + protected readonly TSyntax Stmt; + + protected override CSharpSyntaxNode GetStatementSyntax() => Stmt; + + protected Statement(Context cx, TSyntax stmt, Kinds.StmtKind kind, IStatementParentEntity parent, int child, Location location) + : base(cx) + { + Stmt = stmt; + cx.BindComments(this, location.symbol); + cx.Emit(Tuples.statements(this, kind)); + if (parent.IsTopLevelParent) + cx.Emit(Tuples.stmt_parent_top_level(this, child, parent)); + else + cx.Emit(Tuples.stmt_parent(this, child, parent)); + cx.Emit(Tuples.stmt_location(this, location)); + } + + protected Statement(Context cx, TSyntax stmt, Kinds.StmtKind kind, IStatementParentEntity parent, int child) + : this(cx, stmt, kind, parent, child, cx.Create(stmt.FixedLocation())) { } + + /// + /// Populates statement-type specific relations in the trap file. The general relations + /// statements and stmt_location are populated by the constructor + /// (should not fail), so even if statement-type specific population fails (e.g., in + /// standalone extraction), the statement created via + /// will still + /// be valid. + /// + protected abstract void Populate(); + + protected void TryPopulate() + { + cx.Try(Stmt, null, Populate); + } + + public override string ToString() => Label.ToString(); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Block.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Block.cs new file mode 100644 index 00000000000..575dbab2e4e --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Block.cs @@ -0,0 +1,28 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Block : Statement + { + Block(Context cx, BlockSyntax block, IStatementParentEntity parent, int child) + : base(cx, block, StmtKind.BLOCK, parent, child) { } + + public static Block Create(Context cx, BlockSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Block(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + var child = 0; + foreach (var childStmt in Stmt.Statements.Select(c => Statement.Create(cx, c, this, child))) + { + child += childStmt.NumberOfStatements; + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Break.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Break.cs new file mode 100644 index 00000000000..57141169805 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Break.cs @@ -0,0 +1,20 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Break : Statement + { + Break(Context cx, BreakStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.BREAK, parent, child) { } + + public static Break Create(Context cx, BreakStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Break(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() { } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Case.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Case.cs new file mode 100644 index 00000000000..088b60ee169 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Case.cs @@ -0,0 +1,105 @@ +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + abstract class Case : Statement where TSyntax : SwitchLabelSyntax + { + protected Case(Context cx, TSyntax node, Switch parent, int child) + : base(cx, node, StmtKind.CASE, parent, child, cx.Create(node.GetLocation())) { } + + public static Statement Create(Context cx, SwitchLabelSyntax node, Switch parent, int child) + { + switch (node.Kind()) + { + case SyntaxKind.CaseSwitchLabel: + return CaseLabel.Create(cx, (CaseSwitchLabelSyntax)node, parent, child); + case SyntaxKind.DefaultSwitchLabel: + return CaseDefault.Create(cx, (DefaultSwitchLabelSyntax)node, parent, child); + case SyntaxKind.CasePatternSwitchLabel: + return CasePattern.Create(cx, (CasePatternSwitchLabelSyntax)node, parent, child); + default: + throw new InternalError(node, "Unhandled case label"); + } + } + } + + class CaseLabel : Case + { + CaseLabel(Context cx, CaseSwitchLabelSyntax node, Switch parent, int child) + : base(cx, node, parent, child) { } + + protected override void Populate() + { + var value = Stmt.Value; + Expression.Create(cx, value, this, 0); + Switch.LabelForValue(cx.Model(Stmt).GetConstantValue(value).Value); + } + + public static CaseLabel Create(Context cx, CaseSwitchLabelSyntax node, Switch parent, int child) + { + var ret = new CaseLabel(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + } + + class CaseDefault : Case + { + CaseDefault(Context cx, DefaultSwitchLabelSyntax node, Switch parent, int child) + : base(cx, node, parent, child) { } + + protected override void Populate() { } + + public static CaseDefault Create(Context cx, DefaultSwitchLabelSyntax node, Switch parent, int child) + { + var ret = new CaseDefault(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + } + + class CasePattern : Case + { + CasePattern(Context cx, CasePatternSwitchLabelSyntax node, Switch parent, int child) + : base(cx, node, parent, child) { } + + protected override void Populate() + { + switch(Stmt.Pattern) + { + case DeclarationPatternSyntax declarationPattern: + var symbol = cx.Model(Stmt).GetDeclaredSymbol(declarationPattern.Designation) as ILocalSymbol; + if (symbol != null) + { + var type = Type.Create(cx, symbol.Type); + var isVar = declarationPattern.Type.IsVar; + Expressions.VariableDeclaration.Create(cx, symbol, type, cx.Create(declarationPattern.GetLocation()), cx.Create(declarationPattern.Designation.GetLocation()), isVar, this, 0); + } + + Expressions.TypeAccess.Create(cx, declarationPattern.Type, this, 1); + break; + case ConstantPatternSyntax pattern: + Expression.Create(cx, pattern.Expression, this, 0); + break; + default: + throw new InternalError(Stmt, "Case pattern not handled"); + } + + if (Stmt.WhenClause != null) + { + Expression.Create(cx, Stmt.WhenClause.Condition, this, 2); + } + } + + public static CasePattern Create(Context cx, CasePatternSwitchLabelSyntax node, Switch parent, int child) + { + var ret = new CasePattern(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Catch.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Catch.cs new file mode 100644 index 00000000000..f85e0659dba --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Catch.cs @@ -0,0 +1,51 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Catch : Statement + { + static readonly string SystemExceptionName = typeof(System.Exception).ToString(); + + Catch(Context cx, CatchClauseSyntax node, Try parent, int child) + : base(cx, node, StmtKind.CATCH, parent, child, cx.Create(node.GetLocation())) { } + + protected override void Populate() + { + bool isSpecificCatchClause = Stmt.Declaration != null; + bool hasVariableDeclaration = isSpecificCatchClause && Stmt.Declaration.Identifier.RawKind != 0; + + if (hasVariableDeclaration) // A catch clause of the form 'catch(Ex ex) { ... }' + { + var decl = Expressions.VariableDeclaration.Create(cx, Stmt.Declaration, false, this, 0); + cx.Emit(Tuples.catch_type(this, decl.Type.TypeRef, true)); + } + else if (isSpecificCatchClause) // A catch clause of the form 'catch(Ex) { ... }' + { + cx.Emit(Tuples.catch_type(this, Type.Create(cx, cx.GetType(Stmt.Declaration.Type)).TypeRef, true)); + } + else // A catch clause of the form 'catch { ... }' + { + var exception = Type.Create(cx, cx.Compilation.GetTypeByMetadataName(SystemExceptionName)); + cx.Emit(Tuples.catch_type(this, exception, false)); + } + + if (Stmt.Filter != null) + { + // For backward compatibility, the catch filter clause is child number 2. + Expression.Create(cx, Stmt.Filter.FilterExpression, this, 2); + } + + Create(cx, Stmt.Block, this, 1); + } + + public static Catch Create(Context cx, CatchClauseSyntax node, Try parent, int child) + { + var ret = new Catch(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Checked.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Checked.cs new file mode 100644 index 00000000000..55ce9d80058 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Checked.cs @@ -0,0 +1,23 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Checked : Statement + { + Checked(Context cx, CheckedStatementSyntax stmt, IStatementParentEntity parent, int child) + : base(cx, stmt, StmtKind.CHECKED, parent, child) { } + + public static Checked Create(Context cx, CheckedStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Checked(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + Statement.Create(cx, Stmt.Block, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Continue.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Continue.cs new file mode 100644 index 00000000000..70038e239ff --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Continue.cs @@ -0,0 +1,21 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Continue : Statement + { + Continue(Context cx, ContinueStatementSyntax stmt, IStatementParentEntity parent, int child) + : base(cx, stmt, StmtKind.CONTINUE, parent, child) { } + + public static Continue Create(Context cx, ContinueStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Continue(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() { } + } +} + diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Do.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Do.cs new file mode 100644 index 00000000000..56cbb0a22e1 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Do.cs @@ -0,0 +1,25 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Entities; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Do : Statement + { + Do(Context cx, DoStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.DO, parent, child, cx.Create(node.GetLocation())) { } + + public static Do Create(Context cx, DoStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Do(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + Create(cx, Stmt.Statement, this, 1); + Expression.Create(cx, Stmt.Condition, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Empty.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Empty.cs new file mode 100644 index 00000000000..7b32ec7d09d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Empty.cs @@ -0,0 +1,20 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Empty : Statement + { + Empty(Context cx, EmptyStatementSyntax block, IStatementParentEntity parent, int child) + : base(cx, block, StmtKind.EMPTY, parent, child) { } + + public static Empty Create(Context cx, EmptyStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Empty(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() { } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/ExpressionStatement.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/ExpressionStatement.cs new file mode 100644 index 00000000000..de9a1985987 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/ExpressionStatement.cs @@ -0,0 +1,25 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class ExpressionStatement : Statement + { + ExpressionStatement(Context cx, ExpressionStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, Kinds.StmtKind.EXPR, parent, child) { } + + public static ExpressionStatement Create(Context cx, ExpressionStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new ExpressionStatement(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + if (Stmt.Expression != null) + Expression.Create(cx, Stmt.Expression, this, 0); + else + cx.ModelError(Stmt, "Invalid expression statement"); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Factory.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Factory.cs new file mode 100644 index 00000000000..a804b044506 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Factory.cs @@ -0,0 +1,73 @@ +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + static class Factory + { + internal static Statement Create(Context cx, StatementSyntax node, Statement parent, int child) + { + switch (node.Kind()) + { + case SyntaxKind.ForStatement: + return For.Create(cx, (ForStatementSyntax)node, parent, child); + case SyntaxKind.ExpressionStatement: + return ExpressionStatement.Create(cx, (ExpressionStatementSyntax)node, parent, child); + case SyntaxKind.UsingStatement: + return Using.Create(cx, (UsingStatementSyntax)node, parent, child); + case SyntaxKind.LocalDeclarationStatement: + return LocalDeclaration.Create(cx, (LocalDeclarationStatementSyntax)node, parent, child); + case SyntaxKind.Block: + return Block.Create(cx, (BlockSyntax)node, parent, child); + case SyntaxKind.ReturnStatement: + return Return.Create(cx, (ReturnStatementSyntax)node, parent, child); + case SyntaxKind.SwitchStatement: + return Switch.Create(cx, (SwitchStatementSyntax)node, parent, child); + case SyntaxKind.BreakStatement: + return Break.Create(cx, (BreakStatementSyntax)node, parent, child); + case SyntaxKind.IfStatement: + return If.Create(cx, (IfStatementSyntax)node, parent, child); + case SyntaxKind.WhileStatement: + return While.Create(cx, (WhileStatementSyntax)node, parent, child); + case SyntaxKind.DoStatement: + return Do.Create(cx, (DoStatementSyntax)node, parent, child); + case SyntaxKind.YieldReturnStatement: + return Yield.Create(cx, (YieldStatementSyntax)node, parent, child); + case SyntaxKind.ThrowStatement: + return Throw.Create(cx, (ThrowStatementSyntax)node, parent, child); + case SyntaxKind.TryStatement: + return Try.Create(cx, (TryStatementSyntax)node, parent, child); + case SyntaxKind.EmptyStatement: + return Empty.Create(cx, (EmptyStatementSyntax)node, parent, child); + case SyntaxKind.FixedStatement: + return Fixed.Create(cx, (FixedStatementSyntax)node, parent, child); + case SyntaxKind.LockStatement: + return Lock.Create(cx, (LockStatementSyntax)node, parent, child); + case SyntaxKind.GotoDefaultStatement: + case SyntaxKind.GotoStatement: + case SyntaxKind.GotoCaseStatement: + return Goto.Create(cx, (GotoStatementSyntax)node, parent, child); + case SyntaxKind.LabeledStatement: + return Labeled.Create(cx, (LabeledStatementSyntax)node, parent, child); + case SyntaxKind.CheckedStatement: + return Checked.Create(cx, (CheckedStatementSyntax)node, parent, child); + case SyntaxKind.UncheckedStatement: + return Unchecked.Create(cx, (CheckedStatementSyntax)node, parent, child); + case SyntaxKind.ForEachStatement: + return ForEach.Create(cx, (ForEachStatementSyntax)node, parent, child); + case SyntaxKind.YieldBreakStatement: + return Yield.Create(cx, (YieldStatementSyntax)node, parent, child); + case SyntaxKind.ContinueStatement: + return Continue.Create(cx, (ContinueStatementSyntax)node, parent, child); + case SyntaxKind.UnsafeStatement: + return Unsafe.Create(cx, (UnsafeStatementSyntax)node, parent, child); + case SyntaxKind.LocalFunctionStatement: + return LocalFunction.Create(cx, (LocalFunctionStatementSyntax)node, parent, child); + case SyntaxKind.ForEachVariableStatement: + return ForEachVariable.Create(cx, (ForEachVariableStatementSyntax)node, parent, child); + default: + throw new InternalError(node, "Unhandled statement of kind '{0}'", node.Kind()); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Fixed.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Fixed.cs new file mode 100644 index 00000000000..4e9e4ae4ac9 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Fixed.cs @@ -0,0 +1,25 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Entities.Expressions; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Fixed : Statement + { + Fixed(Context cx, FixedStatementSyntax @fixed, IStatementParentEntity parent, int child) + : base(cx, @fixed, StmtKind.FIXED, parent, child) { } + + public static Fixed Create(Context cx, FixedStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Fixed(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + VariableDeclarations.Populate(cx, Stmt.Declaration, this, -1, childIncrement: -1); + Create(cx, Stmt.Statement, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/For.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/For.cs new file mode 100644 index 00000000000..2c918af83ef --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/For.cs @@ -0,0 +1,45 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Entities.Expressions; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class For : Statement + { + For(Context cx, ForStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.FOR, parent, child) { } + + public static For Create(Context cx, ForStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new For(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + int child = -1; + + if (Stmt.Declaration != null) + VariableDeclarations.Populate(cx, Stmt.Declaration, this, child, childIncrement: -1); + + foreach (var init in Stmt.Initializers) + { + Expression.Create(cx, init, this, child--); + } + + Statement.Create(cx, Stmt.Statement, this, 1 + Stmt.Incrementors.Count); + + child = 1; + foreach (var inc in Stmt.Incrementors) + { + Expression.Create(cx, inc, this, child++); + } + + if (Stmt.Condition != null) + { + Expression.Create(cx, Stmt.Condition, this, 0); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/ForEach.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/ForEach.cs new file mode 100644 index 00000000000..2c3076757c2 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/ForEach.cs @@ -0,0 +1,56 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis.CSharp; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class ForEach : Statement + { + ForEach(Context cx, ForEachStatementSyntax stmt, IStatementParentEntity parent, int child) + : base(cx, stmt, StmtKind.FOREACH, parent, child) { } + + public static ForEach Create(Context cx, ForEachStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new ForEach(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + Expression.Create(cx, Stmt.Expression, this, 1); + + var typeSymbol = cx.Model(Stmt).GetDeclaredSymbol(Stmt); + var type = Type.Create(cx, typeSymbol.Type); + + var location = cx.Create(Stmt.Identifier.GetLocation()); + + if (typeSymbol.Name != "_") + Expressions.VariableDeclaration.Create(cx, typeSymbol, type, location, location, Stmt.Type.IsVar, this, 0); + TypeMention.Create(cx, Stmt.Type, this, type); + + Statement.Create(cx, Stmt.Statement, this, 2); + } + } + + class ForEachVariable : Statement + { + ForEachVariable(Context cx, ForEachVariableStatementSyntax stmt, IStatementParentEntity parent, int child) + : base(cx, stmt, StmtKind.FOREACH, parent, child) { } + + public static ForEachVariable Create(Context cx, ForEachVariableStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new ForEachVariable(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + Expression.Create(cx, Stmt.Variable, this, 0); + Expression.Create(cx, Stmt.Expression, this, 1); + Statement.Create(cx, Stmt.Statement, this, 2); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Goto.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Goto.cs new file mode 100644 index 00000000000..e5fdb2f637f --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Goto.cs @@ -0,0 +1,57 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis.CSharp; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + /// + /// A goto, goto case or goto default. + /// + class Goto : Statement + { + static StmtKind GetKind(GotoStatementSyntax node) + { + switch (node.CaseOrDefaultKeyword.Kind()) + { + case SyntaxKind.None: return StmtKind.GOTO; + case SyntaxKind.DefaultKeyword: return StmtKind.GOTO_DEFAULT; + case SyntaxKind.CaseKeyword: return StmtKind.GOTO_CASE; + default: throw new InternalError(node, "Unhandled goto statement kind {0}", node.CaseOrDefaultKeyword.Kind()); + } + } + + Goto(Context cx, GotoStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, GetKind(node), parent, child) { } + + public static Goto Create(Context cx, GotoStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Goto(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + switch (GetKind(Stmt)) + { + case StmtKind.GOTO: + var target = ((IdentifierNameSyntax)Stmt.Expression).Identifier.Text; + cx.Emit(Tuples.exprorstmt_name(this, target)); + break; + case StmtKind.GOTO_CASE: + Expr = Expression.Create(cx, Stmt.Expression, this, 0); + ConstantValue = Switch.LabelForValue(cx.Model(Stmt).GetConstantValue(Stmt.Expression).Value); + break; + case StmtKind.GOTO_DEFAULT: + ConstantValue = Switch.DefaultLabel; + break; + } + } + + public Expression Expr { get; private set; } + + public object ConstantValue { get; private set; } + + public bool IsDefault => ConstantValue == Switch.DefaultLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/If.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/If.cs new file mode 100644 index 00000000000..6cdd486c363 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/If.cs @@ -0,0 +1,28 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class If : Statement + { + If(Context cx, IfStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.IF, parent, child) { } + + public static If Create(Context cx, IfStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new If(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + Expression.Create(cx, Stmt.Condition, this, 0); + + Create(cx, Stmt.Statement, this, 1); + + if (Stmt.Else != null) + Create(cx, Stmt.Else.Statement, this, 2); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Labeled.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Labeled.cs new file mode 100644 index 00000000000..30c4726c52e --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Labeled.cs @@ -0,0 +1,38 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Labeled : Statement + { + readonly Statement Parent; + readonly int Child; + + Labeled(Context cx, LabeledStatementSyntax stmt, Statement parent, int child) + : base(cx, stmt, StmtKind.LABEL, parent, child) + { + Parent = parent; + Child = child; + } + + public static Labeled Create(Context cx, LabeledStatementSyntax node, Statement parent, int child) + { + var ret = new Labeled(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + cx.Emit(Tuples.exprorstmt_name(this, Stmt.Identifier.ToString())); + + // For compatilibty with the Mono extractor, make insert the labelled statement into the same block + // as this one. The parent MUST be a block statement. + labelledStmt = Statement.Create(cx, Stmt.Statement, Parent, Child + 1); + } + + Statement labelledStmt; + + public override int NumberOfStatements => 1 + labelledStmt.NumberOfStatements; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/LocalDeclaration.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/LocalDeclaration.cs new file mode 100644 index 00000000000..671b4d7d99e --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/LocalDeclaration.cs @@ -0,0 +1,25 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Entities.Expressions; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class LocalDeclaration : Statement + { + LocalDeclaration(Context cx, LocalDeclarationStatementSyntax declStmt, IStatementParentEntity parent, int child) + : base(cx, declStmt, declStmt.IsConst ? StmtKind.CONST_DECL : StmtKind.VAR_DECL, parent, child) { } + + public static LocalDeclaration Create(Context cx, LocalDeclarationStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new LocalDeclaration(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + VariableDeclarations.Populate(cx, Stmt.Declaration, this, 0); + cx.BindComments(this, Stmt.GetLocation()); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/LocalFunction.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/LocalFunction.cs new file mode 100644 index 00000000000..e4a553f4400 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/LocalFunction.cs @@ -0,0 +1,48 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class LocalFunction : Statement + { + LocalFunction(Context cx, LocalFunctionStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.LOCAL_FUNCTION, parent, child, cx.Create(node.GetLocation())) { } + + public static LocalFunction Create(Context cx, LocalFunctionStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new LocalFunction(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + /// + /// Gets the IMethodSymbol for this local function statement. + /// + IMethodSymbol Symbol + { + get + { + // Ideally model.GetDeclaredSymbol(Stmt) would do + // the right thing but it doesn't exist. + // So instead, we have to do the lookup via GetEnclosingSymbol. + + var m = cx.Model(Stmt); + var body = Stmt.Body == null ? Stmt.ExpressionBody : (CSharpSyntaxNode)Stmt.Body; + return m.GetEnclosingSymbol(body.GetLocation().SourceSpan.Start) as IMethodSymbol; + } + } + + /// + /// Gets the function defined by this local statement. + /// + Entities.LocalFunction Function => Entities.LocalFunction.Create(cx, Symbol); + + protected override void Populate() + { + cx.Emit(Tuples.local_function_stmts(this, Function)); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Lock.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Lock.cs new file mode 100644 index 00000000000..e916f9ba601 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Lock.cs @@ -0,0 +1,24 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Lock : Statement + { + Lock(Context cx, LockStatementSyntax @lock, IStatementParentEntity parent, int child) + : base(cx, @lock, StmtKind.LOCK, parent, child) { } + + public static Lock Create(Context cx, LockStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Lock(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + Expression.Create(cx, Stmt.Expression, this, 0); + Statement.Create(cx, Stmt.Statement, this, 1); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Return.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Return.cs new file mode 100644 index 00000000000..2616b874997 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Return.cs @@ -0,0 +1,24 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Return : Statement + { + Return(Context cx, ReturnStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.RETURN, parent, child) { } + + public static Return Create(Context cx, ReturnStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Return(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + if (Stmt.Expression != null) + Expression.Create(cx, Stmt.Expression, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Switch.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Switch.cs new file mode 100644 index 00000000000..06e5364a84a --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Switch.cs @@ -0,0 +1,49 @@ +using System.Linq; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Switch : Statement + { + static readonly object NullLabel = new object(); + public static readonly object DefaultLabel = new object(); + + // Sometimes, the literal "null" is used as a label. + // This is inconveniently represented by the "null" object. + // This cannot be stored in a Dictionary<>, so substitute an object which can be. + public static object LabelForValue(object label) + { + return label ?? NullLabel; + } + + Switch(Context cx, SwitchStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.SWITCH, parent, child) { } + + public static Switch Create(Context cx, SwitchStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Switch(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + Expression.Create(cx, Stmt.Expression, this, 0); + int childIndex = 0; + + foreach (var section in Stmt.Sections) + { + foreach (var stmt in section.Labels.Select(label => Case.Create(cx, label, this, childIndex))) + { + childIndex += stmt.NumberOfStatements; + } + + foreach (var stmt in section.Statements.Select(s => Create(cx, s, this, childIndex))) + { + childIndex += stmt.NumberOfStatements; + } + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Throw.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Throw.cs new file mode 100644 index 00000000000..98e3c37c1dc --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Throw.cs @@ -0,0 +1,24 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Throw : Statement + { + Throw(Context cx, ThrowStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.THROW, parent, child) { } + + public static Throw Create(Context cx, ThrowStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Throw(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + if (Stmt.Expression != null) + Expression.Create(cx, Stmt.Expression, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Try.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Try.cs new file mode 100644 index 00000000000..4c0134839d9 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Try.cs @@ -0,0 +1,47 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Try : Statement + { + Try(Context cx, TryStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.TRY, parent, child) { } + + public static Try Create(Context cx, TryStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Try(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + var child = 1; + foreach (var c in Stmt.Catches) + { + Catch.Create(cx, c, this, child++); + } + + Create(cx, Stmt.Block, this, 0); + + if (Stmt.Finally != null) + { + Create(cx, Stmt.Finally.Block, this, -1); + } + } + + public static SyntaxNodeOrToken NextNode(SyntaxNode node) + { + for (var i = node.Parent.ChildNodesAndTokens().GetEnumerator(); i.MoveNext();) + { + if (i.Current == node) + { + return i.MoveNext() ? i.Current : null; + } + } + return null; + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Unchecked.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Unchecked.cs new file mode 100644 index 00000000000..797d4021708 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Unchecked.cs @@ -0,0 +1,23 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Unchecked : Statement + { + Unchecked(Context cx, CheckedStatementSyntax stmt, IStatementParentEntity parent, int child) + : base(cx, stmt, StmtKind.UNCHECKED, parent, child) { } + + public static Unchecked Create(Context cx, CheckedStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Unchecked(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + Statement.Create(cx, Stmt.Block, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Unsafe.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Unsafe.cs new file mode 100644 index 00000000000..55385c4e604 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Unsafe.cs @@ -0,0 +1,23 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Unsafe : Statement + { + Unsafe(Context cx, UnsafeStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.UNSAFE, parent, child) { } + + public static Unsafe Create(Context cx, UnsafeStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Unsafe(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + Create(cx, Stmt.Block, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Using.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Using.cs new file mode 100644 index 00000000000..596de684de2 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Using.cs @@ -0,0 +1,31 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Entities.Expressions; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Using : Statement + { + Using(Context cx, UsingStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.USING, parent, child) { } + + public static Using Create(Context cx, UsingStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Using(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + if (Stmt.Declaration != null) + VariableDeclarations.Populate(cx, Stmt.Declaration, this, -1, childIncrement: -1); + + if (Stmt.Expression != null) + Expression.Create(cx, Stmt.Expression, this, 0); + + if (Stmt.Statement != null) + Statement.Create(cx, Stmt.Statement, this, 1); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/While.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/While.cs new file mode 100644 index 00000000000..40ba9190b33 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/While.cs @@ -0,0 +1,24 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class While : Statement + { + While(Context cx, WhileStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.WHILE, parent, child) { } + + public static While Create(Context cx, WhileStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new While(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + Expression.Create(cx, Stmt.Condition, this, 0); + Create(cx, Stmt.Statement, this, 1); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Yield.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Yield.cs new file mode 100644 index 00000000000..a1d45af7de2 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Yield.cs @@ -0,0 +1,26 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Yield : Statement + { + Yield(Context cx, YieldStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.YIELD, parent, child) { } + + public static Yield Create(Context cx, YieldStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Yield(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + if (Stmt.Expression != null) + { + Expression.Create(cx, Stmt.Expression, this, 0); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Symbol.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Symbol.cs new file mode 100644 index 00000000000..0c56855db81 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Symbol.cs @@ -0,0 +1,102 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Entities; +using System.Collections.Generic; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + public abstract class CachedSymbol : CachedEntity where T : ISymbol + { + public CachedSymbol(Context cx, T init) + : base(cx, init) { } + + public virtual Type ContainingType => symbol.ContainingType != null ? Type.Create(Context, symbol.ContainingType) : null; + + public void ExtractModifiers() + { + Modifier.ExtractModifiers(Context, this, symbol); + } + + protected void ExtractAttributes() + { + // Only extract attributes for source declarations + if (ReferenceEquals(symbol, symbol.OriginalDefinition)) + Attribute.ExtractAttributes(Context, symbol, this); + } + + protected void ExtractCompilerGenerated() + { + if (symbol.IsImplicitlyDeclared) + Context.Emit(Tuples.compiler_generated(this)); + } + + /// + /// The location which is stored in the database and is used when highlighing source code. + /// It's generally short, e.g. a method name. + /// + public override Microsoft.CodeAnalysis.Location ReportingLocation => symbol.Locations.FirstOrDefault(); + + /// + /// The full text span of the entity, e.g. for binding comments. + /// + public virtual Microsoft.CodeAnalysis.Location FullLocation => symbol.Locations.FirstOrDefault(); + + public virtual IEnumerable Locations + { + get + { + var loc = ReportingLocation; + if (loc != null) + { + // Some built in operators lack locations, so loc is null. + yield return Context.Create(ReportingLocation); + if (Context.Extractor.OutputPath != null && loc.Kind == LocationKind.SourceFile) + yield return Assembly.CreateOutputAssembly(Context); + } + } + } + + /// + /// Bind comments to this symbol. + /// Comments are only bound to source declarations. + /// + protected void BindComments() + { + if (!symbol.IsImplicitlyDeclared && IsSourceDeclaration && symbol.FromSource()) + Context.BindComments(this, FullLocation); + } + + public BlockSyntax Block + { + get + { + return symbol. + DeclaringSyntaxReferences. + Select(r => r.GetSyntax()). + SelectMany(n => n.ChildNodes()). + OfType(). + FirstOrDefault(); + } + } + + public ExpressionSyntax ExpressionBody + { + get + { + return symbol. + DeclaringSyntaxReferences. + SelectMany(r => r.GetSyntax().ChildNodes()). + OfType(). + Select(arrow => arrow.Expression). + FirstOrDefault(); + } + } + + public virtual bool IsSourceDeclaration => symbol.IsSourceDeclaration(); + + public override bool NeedsPopulation => Context.Defines(symbol); + + public Extraction.Entities.Location Location => Context.Create(ReportingLocation); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/TypeMention.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/TypeMention.cs new file mode 100644 index 00000000000..9b32bc359e8 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/TypeMention.cs @@ -0,0 +1,93 @@ +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Entities; +using Semmle.Util; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class TypeMention : FreshEntity + { + readonly TypeSyntax Syntax; + readonly IEntity Parent; + readonly Type Type; + readonly Microsoft.CodeAnalysis.Location Loc; + + TypeMention(Context cx, TypeSyntax syntax, IEntity parent, Type type, Microsoft.CodeAnalysis.Location loc = null) + : base(cx) + { + Syntax = syntax; + Parent = parent; + Type = type; + Loc = loc; + } + + void Populate() + { + switch (Syntax.Kind()) + { + case SyntaxKind.ArrayType: + var ats = (ArrayTypeSyntax)Syntax; + var at = (ArrayType)Type; + Emit(Loc ?? Syntax.GetLocation(), Parent, Type); + Create(cx, ats.ElementType, this, at.ElementType); + return; + case SyntaxKind.NullableType: + var nts = (NullableTypeSyntax)Syntax; + var nt = (NamedType)Type; + Emit(Loc ?? Syntax.GetLocation(), Parent, Type); + Create(cx, nts.ElementType, this, nt.TypeArguments[0]); + return; + case SyntaxKind.TupleType: + var tts = (TupleTypeSyntax)Syntax; + var tt = (TupleType)Type; + Emit(Loc ?? Syntax.GetLocation(), Parent, Type); + tts.Elements.Zip(tt.TupleElements, (s, t) => Create(cx, s.Type, this, t.Type)).Enumerate(); + return; + case SyntaxKind.PointerType: + var pts = (PointerTypeSyntax)Syntax; + var pt = (PointerType)Type; + Emit(Loc ?? Syntax.GetLocation(), Parent, Type); + Create(cx, pts.ElementType, this, pt.PointedAtType); + return; + case SyntaxKind.GenericName: + var gns = (GenericNameSyntax)Syntax; + Emit(Loc ?? gns.Identifier.GetLocation(), Parent, Type); + gns.TypeArgumentList.Arguments.Zip(Type.TypeMentions, (s, t) => Create(cx, s, this, t)).Enumerate(); + return; + case SyntaxKind.QualifiedName: + if (Type.ContainingType == null) + { + // namespace qualifier + Emit(Loc ?? Syntax.GetLocation(), Parent, Type); + } + else + { + // Type qualifier + var qns = (QualifiedNameSyntax)Syntax; + var right = Create(cx, qns.Right, Parent, Type); + Create(cx, qns.Left, right, Type.ContainingType); + } + return; + default: + Emit(Loc ?? Syntax.GetLocation(), Parent, Type); + return; + } + } + + void Emit(Microsoft.CodeAnalysis.Location loc, IEntity parent, Type type) + { + cx.Emit(Tuples.type_mention(this, type.TypeRef, parent)); + cx.Emit(Tuples.type_mention_location(this, cx.Create(loc))); + } + + public static TypeMention Create(Context cx, TypeSyntax syntax, IEntity parent, Type type, Microsoft.CodeAnalysis.Location loc = null) + { + var ret = new TypeMention(cx, syntax, parent, type, loc); + cx.Try(syntax, null, () => ret.Populate()); + return ret; + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.OptionalLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/ArrayType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/ArrayType.cs new file mode 100644 index 00000000000..e791c4b27db --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/ArrayType.cs @@ -0,0 +1,53 @@ +using Microsoft.CodeAnalysis; + +namespace Semmle.Extraction.CSharp.Entities +{ + class ArrayType : Type + { + ArrayType(Context cx, IArrayTypeSymbol init) + : base(cx, init) + { + element = Create(cx, symbol.ElementType); + } + + readonly Type element; + + public int Rank => symbol.Rank; + + public override Type ElementType => element; + + public override int Dimension => 1 + element.Dimension; + + // All array types are extracted because they won't + // be extracted in their defining assembly. + public override bool NeedsPopulation => true; + + public override void Populate() + { + Context.Emit(Tuples.array_element_type(this, Dimension, Rank, element.TypeRef)); + ExtractType(); + } + + public override IId Id + { + get + { + return new Key(tb => + { + tb.Append(element); + symbol.BuildArraySuffix(tb); + tb.Append(";type"); + }); + } + } + + public static ArrayType Create(Context cx, IArrayTypeSymbol symbol) => ArrayTypeFactory.Instance.CreateEntity(cx, symbol); + + class ArrayTypeFactory : ICachedEntityFactory + { + public static readonly ArrayTypeFactory Instance = new ArrayTypeFactory(); + + public ArrayType Create(Context cx, IArrayTypeSymbol init) => new ArrayType(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/DynamicType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/DynamicType.cs new file mode 100644 index 00000000000..8c1f95bd423 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/DynamicType.cs @@ -0,0 +1,35 @@ +using Microsoft.CodeAnalysis; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class DynamicType : Type + { + DynamicType(Context cx, IDynamicTypeSymbol init) + : base(cx, init) { } + + public static DynamicType Create(Context cx, IDynamicTypeSymbol type) => DynamicTypeFactory.Instance.CreateEntity(cx, type); + + public override Microsoft.CodeAnalysis.Location ReportingLocation => Context.Compilation.ObjectType.Locations.FirstOrDefault(); + + public override void Populate() + { + Context.Emit(Tuples.types(this, Kinds.TypeKind.DYNAMIC, "dynamic")); + Context.Emit(Tuples.type_location(this, Location)); + + Context.Emit(Tuples.has_modifiers(this, Modifier.Create(Context, "public"))); + Context.Emit(Tuples.parent_namespace(this, Namespace.Create(Context, Context.Compilation.GlobalNamespace))); + } + + static readonly Key id = new Key("dynamic;type"); + + public override IId Id => id; + + class DynamicTypeFactory : ICachedEntityFactory + { + public static readonly DynamicTypeFactory Instance = new DynamicTypeFactory(); + + public DynamicType Create(Context cx, IDynamicTypeSymbol init) => new DynamicType(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs new file mode 100644 index 00000000000..a5dba66c934 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs @@ -0,0 +1,204 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Entities; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class NamedType : Type + { + NamedType(Context cx, INamedTypeSymbol init) + : base(cx, init) + { + typeArgumentsLazy = new Lazy(() => symbol.TypeArguments.Select(t => Create(cx, t)).ToArray()); + } + + public static NamedType Create(Context cx, INamedTypeSymbol type) => NamedTypeFactory.Instance.CreateEntity(cx, type); + + public override bool NeedsPopulation => base.NeedsPopulation || symbol.TypeKind == TypeKind.Error; + + public override void Populate() + { + if (symbol.TypeKind == TypeKind.Error) + { + Context.Extractor.MissingType(symbol.ToString()); + return; + } + + Context.Emit(Tuples.typeref_type((NamedTypeRef)TypeRef, this)); + + if (symbol.IsGenericType) + { + if (symbol.IsBoundNullable()) + { + // An instance of Nullable + Context.Emit(Tuples.nullable_underlying_type(this, Create(Context, symbol.TypeArguments[0]).TypeRef)); + } + else if (symbol.IsReallyUnbound()) + { + Context.Emit(Tuples.is_generic(this)); + + for (int i = 0; i < symbol.TypeParameters.Length; ++i) + { + TypeParameter.Create(Context, symbol.TypeParameters[i]); + var param = symbol.TypeParameters[i]; + var typeParameter = TypeParameter.Create(Context, param); + Context.Emit(Tuples.type_parameters(typeParameter, i, this)); + } + } + else + { + Context.Emit(Tuples.is_constructed(this)); + Context.Emit(Tuples.constructed_generic(this, Type.Create(Context, symbol.ConstructedFrom).TypeRef)); + + for (int i = 0; i < symbol.TypeArguments.Length; ++i) + { + Context.Emit(Tuples.type_arguments(TypeArguments[i].TypeRef, i, this)); + } + } + } + + ExtractType(); + + if (symbol.EnumUnderlyingType != null) + { + Context.Emit(Tuples.enum_underlying_type(this, Type.Create(Context, symbol.EnumUnderlyingType).TypeRef)); + } + + // Class location + if (!symbol.IsGenericType || symbol.IsReallyUnbound()) + { + foreach (var l in Locations) + Context.Emit(Tuples.type_location(this, l)); + } + } + + readonly Lazy typeArgumentsLazy; + public Type[] TypeArguments => typeArgumentsLazy.Value; + + public override IEnumerable TypeMentions => TypeArguments; + + public override IEnumerable Locations + { + get + { + foreach (var l in GetLocations(symbol)) + yield return Context.Create(l); + + if (Context.Extractor.OutputPath != null && symbol.DeclaringSyntaxReferences.Any()) + yield return Assembly.CreateOutputAssembly(Context); + } + } + + static IEnumerable GetLocations(INamedTypeSymbol type) + { + return type.Locations. + Where(l => l.IsInMetadata). + Concat( + type. + DeclaringSyntaxReferences. + Select(loc => loc.GetSyntax()). + OfType(). + Select(l => l.FixedLocation()) + ); + } + + public override Microsoft.CodeAnalysis.Location ReportingLocation => GetLocations(symbol).FirstOrDefault(); + + public override IId Id + { + get + { + return new Key(tb => + { + // All syntactic sub terms (types) are referenced by key in the ID + symbol.BuildTypeId(Context, tb, (cx0, tb0, sub) => tb0.Append(Create(cx0, sub))); + tb.Append(";type"); + }); + } + } + + /// + /// Returns the element type in an Enumerable/IEnumerable + /// + /// Extraction context. + /// The enumerable type. + /// The element type, or null. + static ITypeSymbol GetElementType(Context cx, INamedTypeSymbol type) + { + return GetEnumerableType(cx, type) ?? + type.AllInterfaces. + Where(i => i.OriginalDefinition.SpecialType == SpecialType.System_Collections_Generic_IEnumerable_T). + Concat(type.AllInterfaces.Where(i => i.SpecialType == SpecialType.System_Collections_IEnumerable)). + Select(i => GetEnumerableType(cx, i)). + FirstOrDefault(); + } + + static ITypeSymbol GetEnumerableType(Context cx, INamedTypeSymbol type) + { + return type.SpecialType == SpecialType.System_Collections_IEnumerable ? + cx.Compilation.ObjectType : + type.OriginalDefinition.SpecialType == SpecialType.System_Collections_Generic_IEnumerable_T ? + type.TypeArguments.First() : + null; + } + + public override Type ElementType + { + get + { + var elementType = GetElementType(Context, symbol); + return elementType == null ? null : Create(Context, elementType); + } + } + + class NamedTypeFactory : ICachedEntityFactory + { + public static readonly NamedTypeFactory Instance = new NamedTypeFactory(); + + public NamedType Create(Context cx, INamedTypeSymbol init) => new NamedType(cx, init); + } + + public override Type TypeRef => NamedTypeRef.Create(Context, symbol); + } + + class NamedTypeRef : Type + { + readonly Type referencedType; + + public NamedTypeRef(Context cx, INamedTypeSymbol symbol) : base(cx, symbol) + { + referencedType = Type.Create(cx, symbol); + } + + public static NamedTypeRef Create(Context cx, INamedTypeSymbol type) => NamedTypeRefFactory.Instance.CreateEntity(cx, type); + + class NamedTypeRefFactory : ICachedEntityFactory + { + public static readonly NamedTypeRefFactory Instance = new NamedTypeRefFactory(); + + public NamedTypeRef Create(Context cx, INamedTypeSymbol init) => new NamedTypeRef(cx, init); + } + + public override bool NeedsPopulation => true; + + public override IId Id + { + get + { + return new Key(tb => + { + tb.Append(referencedType).Append(";typeref"); + }); + } + } + + public override void Populate() + { + Context.Emit(Tuples.typerefs(this, symbol.Name)); + } + }; +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NullType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NullType.cs new file mode 100644 index 00000000000..55833879224 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NullType.cs @@ -0,0 +1,35 @@ +using Microsoft.CodeAnalysis; + +namespace Semmle.Extraction.CSharp.Entities +{ + class NullType : Type + { + NullType(Context cx) + : base(cx, null) { } + + public override void Populate() + { + Context.Emit(Tuples.types(this, Kinds.TypeKind.NULL, "null")); + } + + public override IId Id => new Key(";type"); + + public override bool NeedsPopulation => true; + + public override int GetHashCode() => 987354; + + public override bool Equals(object obj) + { + return obj != null && obj.GetType() == typeof(NullType); + } + + public static NullType Create(Context cx) => NullTypeFactory.Instance.CreateEntity(cx, null); + + class NullTypeFactory : ICachedEntityFactory + { + public static readonly NullTypeFactory Instance = new NullTypeFactory(); + + public NullType Create(Context cx, ITypeSymbol init) => new NullType(cx); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/PointerType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/PointerType.cs new file mode 100644 index 00000000000..3f722b05e6f --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/PointerType.cs @@ -0,0 +1,36 @@ +using Microsoft.CodeAnalysis; + +namespace Semmle.Extraction.CSharp.Entities +{ + class PointerType : Type + { + PointerType(Context cx, IPointerTypeSymbol init) + : base(cx, init) + { + PointedAtType = Create(cx, symbol.PointedAtType); + } + + public override IId Id => new Key(PointedAtType, "*;type"); + + // All pointer types are extracted because they won't + // be extracted in their defining assembly. + public override bool NeedsPopulation => true; + + public override void Populate() + { + Context.Emit(Tuples.pointer_referent_type(this, PointedAtType.TypeRef)); + ExtractType(); + } + + public Type PointedAtType { get; private set; } + + public static PointerType Create(Context cx, IPointerTypeSymbol symbol) => PointerTypeFactory.Instance.CreateEntity(cx, symbol); + + class PointerTypeFactory : ICachedEntityFactory + { + public static readonly PointerTypeFactory Instance = new PointerTypeFactory(); + + public PointerType Create(Context cx, IPointerTypeSymbol init) => new PointerType(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TupleType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TupleType.cs new file mode 100644 index 00000000000..b28846123e6 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TupleType.cs @@ -0,0 +1,68 @@ +using Microsoft.CodeAnalysis; +using Semmle.Extraction.Entities; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + /// + /// A tuple type, which is a C# type but not a .Net type. + /// Tuples have the underlying type System.ValueTuple. + /// + class TupleType : Type + { + public static TupleType Create(Context cx, INamedTypeSymbol type) => TupleTypeFactory.Instance.CreateEntity(cx, type); + + class TupleTypeFactory : ICachedEntityFactory + { + public static readonly TupleTypeFactory Instance = new TupleTypeFactory(); + + public TupleType Create(Context cx, INamedTypeSymbol init) => new TupleType(cx, init); + } + + TupleType(Context cx, INamedTypeSymbol init) : base(cx, init) + { + tupleElementsLazy = new Lazy(() => symbol.TupleElements.Select(t => Field.Create(cx, t)).ToArray()); + } + + // All tuple types are "local types" + public override bool NeedsPopulation => true; + + public override IId Id + { + get + { + return new Key(tb => + { + symbol.BuildTypeId(Context, tb, (cx0, tb0, sub) => tb0.Append(Create(cx0, sub))); + tb.Append(";tuple"); + }); + } + } + + public override void Populate() + { + ExtractType(); + ExtractGenerics(); + + var underlyingType = NamedType.Create(Context, symbol.TupleUnderlyingType); + Context.Emit(Tuples.tuple_underlying_type(this, underlyingType)); + + int index = 0; + foreach (var element in TupleElements) + Context.Emit(Tuples.tuple_element(this, index++, element)); + + // Note: symbol.Locations seems to be very inconsistent + // about what locations are available for a tuple type. + // Sometimes it's the source code, and sometimes it's empty. + foreach (var l in symbol.Locations) + Context.Emit(Tuples.type_location(this, Context.Create(l))); + } + + readonly Lazy tupleElementsLazy; + public Field[] TupleElements => tupleElementsLazy.Value; + + public override IEnumerable TypeMentions => TupleElements.Select(e => e.Type); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs new file mode 100644 index 00000000000..ac1a7e7f10e --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs @@ -0,0 +1,327 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Util; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + public abstract class Type : CachedSymbol + { + public Type(Context cx, ITypeSymbol init) + : base(cx, init) { } + + public virtual Type ElementType => null; + + public override bool NeedsPopulation => + base.NeedsPopulation || symbol.TypeKind == TypeKind.Dynamic || symbol.TypeKind == TypeKind.TypeParameter; + + public static bool ConstructedOrParentIsConstructed(INamedTypeSymbol symbol) + { + return !Equals(symbol, symbol.OriginalDefinition) || + symbol.ContainingType != null && ConstructedOrParentIsConstructed(symbol.ContainingType); + } + + static Kinds.TypeKind GetClassType(Context cx, ITypeSymbol t) + { + switch (t.SpecialType) + { + case SpecialType.System_Int32: return Kinds.TypeKind.INT; + case SpecialType.System_UInt32: return Kinds.TypeKind.UINT; + case SpecialType.System_Int16: return Kinds.TypeKind.SHORT; + case SpecialType.System_UInt16: return Kinds.TypeKind.USHORT; + case SpecialType.System_UInt64: return Kinds.TypeKind.ULONG; + case SpecialType.System_Int64: return Kinds.TypeKind.LONG; + case SpecialType.System_Void: return Kinds.TypeKind.VOID; + case SpecialType.System_Double: return Kinds.TypeKind.DOUBLE; + case SpecialType.System_Byte: return Kinds.TypeKind.BYTE; + case SpecialType.System_SByte: return Kinds.TypeKind.SBYTE; + case SpecialType.System_Boolean: return Kinds.TypeKind.BOOL; + case SpecialType.System_Char: return Kinds.TypeKind.CHAR; + case SpecialType.System_Decimal: return Kinds.TypeKind.DECIMAL; + case SpecialType.System_Single: return Kinds.TypeKind.FLOAT; + case SpecialType.System_IntPtr: return Kinds.TypeKind.INT_PTR; + default: + if (t.IsBoundNullable()) return Kinds.TypeKind.NULLABLE; + switch (t.TypeKind) + { + case TypeKind.Class: return Kinds.TypeKind.CLASS; + case TypeKind.Struct: + return ((INamedTypeSymbol)t).IsTupleType ? Kinds.TypeKind.TUPLE : Kinds.TypeKind.STRUCT; + case TypeKind.Interface: return Kinds.TypeKind.INTERFACE; + case TypeKind.Array: return Kinds.TypeKind.ARRAY; + case TypeKind.Enum: return Kinds.TypeKind.ENUM; + case TypeKind.Delegate: return Kinds.TypeKind.DELEGATE; + case TypeKind.Pointer: return Kinds.TypeKind.POINTER; + default: + cx.ModelError(t, "Unhandled type kind '{0}'", t.TypeKind); + return Kinds.TypeKind.UNKNOWN; + } + } + } + + class DisplayNameTrapBuilder : ITrapBuilder + { + public readonly List Fragments = new List(); + + public ITrapBuilder Append(object arg) + { + Fragments.Add(arg.ToString()); + return this; + } + + public ITrapBuilder Append(string arg) + { + Fragments.Add(arg); + return this; + } + + public ITrapBuilder AppendLine() + { + throw new NotImplementedException(); + } + } + + protected void ExtractType() + { + ExtractAttributes(); + + var tb = new DisplayNameTrapBuilder(); + symbol.BuildDisplayName(Context, tb); + Context.Emit(Tuples.types(this, GetClassType(Context, symbol), tb.Fragments.ToArray())); + + // Visit base types + var baseTypes = new List(); + if (symbol.BaseType != null) + { + Type baseKey = Create(Context, symbol.BaseType); + Context.Emit(Tuples.extend(this, baseKey.TypeRef)); + if (symbol.TypeKind != TypeKind.Struct) + baseTypes.Add(baseKey); + } + + if (base.symbol.TypeKind == TypeKind.Interface) + { + Context.Emit(Tuples.extend(this, Create(Context, Context.Compilation.ObjectType))); + } + + if (!(base.symbol is IArrayTypeSymbol)) + { + foreach (var t in base.symbol.Interfaces.Select(i=>Create(Context, i))) + { + Context.Emit(Tuples.implement(this, t.TypeRef)); + baseTypes.Add(t); + } + } + + var containingType = ContainingType; + if (containingType != null && symbol.Kind != SymbolKind.TypeParameter) + { + Type originalDefinition = symbol.TypeKind == TypeKind.Error ? this : Create(Context, symbol.OriginalDefinition); + Context.Emit(Tuples.nested_types(this, containingType, originalDefinition)); + } + else if (symbol.ContainingNamespace != null) + { + Context.Emit(Tuples.parent_namespace(this, Namespace.Create(Context, symbol.ContainingNamespace))); + } + + if (symbol is IArrayTypeSymbol) + { + // They are in the namespace of the original object + ITypeSymbol elementType = ((IArrayTypeSymbol)symbol).ElementType; + INamespaceSymbol ns = elementType.TypeKind == TypeKind.TypeParameter ? Context.Compilation.GlobalNamespace : elementType.ContainingNamespace; + if (ns != null) + Context.Emit(Tuples.parent_namespace(this, Namespace.Create(Context, ns))); + } + + if (symbol is IPointerTypeSymbol) + { + ITypeSymbol elementType = ((IPointerTypeSymbol)symbol).PointedAtType; + INamespaceSymbol ns = elementType.TypeKind == TypeKind.TypeParameter ? Context.Compilation.GlobalNamespace : elementType.ContainingNamespace; + + if (ns != null) + Context.Emit(Tuples.parent_namespace(this, Namespace.Create(Context, ns))); + } + + if (symbol.BaseType != null && symbol.BaseType.SpecialType == SpecialType.System_MulticastDelegate) + { + // This is a delegate. + // The method "Invoke" has the return type. + var invokeMethod = ((INamedTypeSymbol)symbol).DelegateInvokeMethod; + + // Copy the parameters from the "Invoke" method to the delegate type + for (var i = 0; i < invokeMethod.Parameters.Length; ++i) + { + var param = invokeMethod.Parameters[i]; + var originalParam = invokeMethod.OriginalDefinition.Parameters[i]; + var originalParamEntity = Equals(param, originalParam) ? null : + DelegateTypeParameter.Create(Context, originalParam, Create(Context, ((INamedTypeSymbol)symbol).ConstructedFrom)); + DelegateTypeParameter.Create(Context, param, this, originalParamEntity); + } + + var returnKey = Create(Context, invokeMethod.ReturnType); + Context.Emit(Tuples.delegate_return_type(this, returnKey.TypeRef)); + if (invokeMethod.ReturnsByRef) + Context.Emit(Tuples.ref_returns(this)); + if (invokeMethod.ReturnsByRefReadonly) + Context.Emit(Tuples.ref_readonly_returns(this)); + } + + Modifier.ExtractModifiers(Context, this, symbol); + + if (IsSourceDeclaration && symbol.FromSource()) + { + var declSyntaxReferences = symbol.DeclaringSyntaxReferences.Select(d => d.GetSyntax()).ToArray(); + + var baseLists = declSyntaxReferences.OfType().Select(c => c.BaseList); + baseLists = baseLists.Concat(declSyntaxReferences.OfType().Select(c => c.BaseList)); + baseLists = baseLists.Concat(declSyntaxReferences.OfType().Select(c => c.BaseList)); + + baseLists. + Where(bl => bl != null). + SelectMany(bl => bl.Types). + Zip(baseTypes.Where(bt => bt.symbol.SpecialType != SpecialType.System_Object), + (s, t) => TypeMention.Create(Context, s.Type, this, t)). + Enumerate(); + } + } + + /// + /// Called to extract all members and nested types. + /// This is called on each member of a namespace, + /// in either source code or an assembly. + /// + public void ExtractRecursive() + { + foreach (var l in symbol.DeclaringSyntaxReferences.Select(s => s.GetSyntax().GetLocation())) + { + Context.BindComments(this, l); + } + + foreach (var member in symbol.GetMembers()) + { + switch (member.Kind) + { + case SymbolKind.NamedType: + Create(Context, (ITypeSymbol)member).ExtractRecursive(); + break; + default: + Context.CreateEntity(member); + break; + } + } + } + + /// + /// Extracts all members and nested types of this type. + /// + public void ExtractGenerics() + { + if (symbol == null || !NeedsPopulation || !Context.ExtractGenerics(this)) + return; + + var members = new List(); + + foreach (var member in symbol.GetMembers()) + members.Add(member); + foreach (var member in symbol.GetTypeMembers()) + members.Add(member); + + // Mono extractor puts all BASE interface members as members of the current interface. + + if (symbol.TypeKind == TypeKind.Interface) + { + foreach (var baseInterface in symbol.Interfaces) + { + foreach (var member in baseInterface.GetMembers()) + members.Add(member); + foreach (var member in baseInterface.GetTypeMembers()) + members.Add(member); + } + } + + foreach (var member in members) + { + Context.CreateEntity(member); + } + + if (symbol.BaseType != null) + Create(Context, symbol.BaseType).ExtractGenerics(); + + foreach (var i in symbol.Interfaces) + { + Create(Context, i).ExtractGenerics(); + } + } + + public void ExtractRecursive(IEntity parent) + { + if (symbol.ContainingSymbol.Kind == SymbolKind.Namespace && !symbol.ContainingNamespace.IsGlobalNamespace) + { + Context.Emit(Tuples.parent_namespace_declaration(this, (NamespaceDeclaration)parent)); + } + + ExtractRecursive(); + } + + public static Type Create(Context cx, ITypeSymbol type) + { + type = cx.DisambiguateType(type); + const bool errorTypeIsNull = false; + return type == null || (errorTypeIsNull && type.TypeKind == TypeKind.Error) ? + NullType.Create(cx) : (Type)cx.CreateEntity(type); + } + + public virtual int Dimension => 0; + + public static bool IsDelegate(ITypeSymbol symbol) => + symbol != null && symbol.TypeKind == TypeKind.Delegate; + + /// + /// A copy of a delegate "Invoke" method parameter used for the delgate + /// type. + /// + class DelegateTypeParameter : Parameter + { + DelegateTypeParameter(Context cx, IParameterSymbol init, IEntity parent, Parameter original) + : base(cx, init, parent, original) { } + + new public static DelegateTypeParameter Create(Context cx, IParameterSymbol param, IEntity parent, Parameter original = null) => + DelegateTypeParameterFactory.Instance.CreateEntity(cx, (param, parent, original)); + + class DelegateTypeParameterFactory : ICachedEntityFactory<(IParameterSymbol, IEntity, Parameter), DelegateTypeParameter> + { + public static readonly DelegateTypeParameterFactory Instance = new DelegateTypeParameterFactory(); + + public DelegateTypeParameter Create(Context cx, (IParameterSymbol, IEntity, Parameter) init) => + new DelegateTypeParameter(cx, init.Item1, init.Item2, init.Item3); + } + } + + /// + /// Gets a reference to this type, if the type + /// is defined in another assembly. + /// + public virtual Type TypeRef => this; + + public virtual IEnumerable TypeMentions + { + get + { + yield break; + } + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } + + abstract class Type : Type where T : ITypeSymbol + { + public Type(Context cx, T init) + : base(cx, init) { } + + public new T symbol => (T)base.symbol; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameter.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameter.cs new file mode 100644 index 00000000000..1ee72f27c13 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameter.cs @@ -0,0 +1,131 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Entities; +using System.Collections.Generic; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + enum Variance + { + None = 0, + Out = 1, + In = 2 + } + + class TypeParameter : Type + { + TypeParameter(Context cx, ITypeParameterSymbol init) + : base(cx, init) { } + + static readonly string valueTypeName = typeof(System.ValueType).ToString(); + + public override void Populate() + { + var constraints = new TypeParameterConstraints(Context); + Context.Emit(Tuples.type_parameter_constraints(constraints, this)); + + if (symbol.HasReferenceTypeConstraint) + Context.Emit(Tuples.general_type_parameter_constraints(constraints, 1)); + + if (symbol.HasValueTypeConstraint) + Context.Emit(Tuples.general_type_parameter_constraints(constraints, 2)); + + if (symbol.HasConstructorConstraint) + Context.Emit(Tuples.general_type_parameter_constraints(constraints, 3)); + + ITypeSymbol baseType = symbol.HasValueTypeConstraint ? + Context.Compilation.GetTypeByMetadataName(valueTypeName) : + Context.Compilation.ObjectType; + + var constraintTypes = new List(); + foreach (var abase in symbol.ConstraintTypes) + { + if (abase.TypeKind != TypeKind.Interface) + baseType = abase; + var t = Create(Context, abase); + Context.Emit(Tuples.specific_type_parameter_constraints(constraints, t.TypeRef)); + constraintTypes.Add(t); + } + + Context.Emit(Tuples.types(this, Semmle.Extraction.Kinds.TypeKind.TYPE_PARAMETER, symbol.Name)); + Context.Emit(Tuples.extend(this, Create(Context, baseType).TypeRef)); + + Namespace parentNs = Namespace.Create(Context, symbol.TypeParameterKind == TypeParameterKind.Method ? Context.Compilation.GlobalNamespace : symbol.ContainingNamespace); + Context.Emit(Tuples.parent_namespace(this, parentNs)); + + foreach (var l in symbol.Locations) + { + Context.Emit(Tuples.type_location(this, Context.Create(l))); + } + + if (this.IsSourceDeclaration) + { + var declSyntaxReferences = symbol.DeclaringSyntaxReferences.Select(d => d.GetSyntax()). + Select(s => s.Parent).Where(p => p != null).Select(p => p.Parent).ToArray(); + var clauses = declSyntaxReferences.OfType().SelectMany(m => m.ConstraintClauses); + clauses = clauses.Concat(declSyntaxReferences.OfType().SelectMany(c => c.ConstraintClauses)); + clauses = clauses.Concat(declSyntaxReferences.OfType().SelectMany(c => c.ConstraintClauses)); + clauses = clauses.Concat(declSyntaxReferences.OfType().SelectMany(c => c.ConstraintClauses)); + int i = 0; + foreach (var clause in clauses.Where(c => c.Name.ToString() == symbol.Name)) + { + TypeMention.Create(Context, clause.Name, this, this); + foreach (var constraint in clause.Constraints.OfType()) + TypeMention.Create(Context, constraint.Type, this, constraintTypes[i++]); + } + } + } + + static public TypeParameter Create(Context cx, ITypeParameterSymbol p) => + TypeParameterFactory.Instance.CreateEntity(cx, p); + + /// + /// The variance of this type parameter. + /// + public Variance Variance + { + get + { + switch (symbol.Variance) + { + case VarianceKind.None: return Variance.None; + case VarianceKind.Out: return Variance.Out; + case VarianceKind.In: return Variance.In; + default: + throw new InternalError("Unexpected VarianceKind {0}", symbol.Variance); + } + } + } + + public override IId Id + { + get + { + string kind; + IEntity containingEntity; + switch (symbol.TypeParameterKind) + { + case TypeParameterKind.Method: + kind = "methodtypeparameter"; + containingEntity = Method.Create(Context, (IMethodSymbol)symbol.ContainingSymbol); + break; + case TypeParameterKind.Type: + kind = "typeparameter"; + containingEntity = Create(Context, symbol.ContainingType); + break; + default: + throw new InternalError(symbol, "Unhandled type parameter kind {0}", symbol.TypeParameterKind); + } + return new Key(containingEntity, "_", symbol.Ordinal, ";", kind); + } + } + + class TypeParameterFactory : ICachedEntityFactory + { + public static readonly TypeParameterFactory Instance = new TypeParameterFactory(); + + public TypeParameter Create(Context cx, ITypeParameterSymbol init) => new TypeParameter(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameterConstraints.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameterConstraints.cs new file mode 100644 index 00000000000..425fdc1fe71 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameterConstraints.cs @@ -0,0 +1,10 @@ +namespace Semmle.Extraction.CSharp.Entities +{ + class TypeParameterConstraints : FreshEntity + { + public TypeParameterConstraints(Context cx) + : base(cx) { } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/UserOperator.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/UserOperator.cs new file mode 100644 index 00000000000..2208eec201d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/UserOperator.cs @@ -0,0 +1,207 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class UserOperator : Method + { + protected UserOperator(Context cx, IMethodSymbol init) + : base(cx, init) { } + + public override void Populate() + { + PopulateMethod(); + ExtractModifiers(); + + var returnType = Type.Create(Context, symbol.ReturnType); + Context.Emit(Tuples.operators(this, + symbol.Name, + OperatorSymbol(Context, symbol.Name), + ContainingType, + returnType.TypeRef, + (UserOperator)OriginalDefinition)); + + foreach (var l in Locations) + Context.Emit(Tuples.operator_location(this, l)); + + if (IsSourceDeclaration) + { + var declSyntaxReferences = symbol.DeclaringSyntaxReferences.Select(s => s.GetSyntax()).ToArray(); + foreach (var declaration in declSyntaxReferences.OfType()) + TypeMention.Create(Context, declaration.ReturnType, this, returnType); + foreach (var declaration in declSyntaxReferences.OfType()) + TypeMention.Create(Context, declaration.Type, this, returnType); + } + + ContainingType.ExtractGenerics(); + } + + public override bool NeedsPopulation => Context.Defines(symbol) || IsImplicitOperator(out _); + + public override Type ContainingType + { + get + { + IsImplicitOperator(out var containingType); + return Type.Create(Context, containingType); + } + } + + public override IId Id + { + get + { + return new Key(tb => + { + AddSignatureTypeToId(Context, tb, symbol, symbol.ReturnType); // Needed for op_explicit(), which differs only by return type. + tb.Append(" "); + BuildMethodId(this, tb); + }); + } + } + + /// + /// For some reason, some operators are missing from the Roslyn database of mscorlib. + /// This method returns true for such operators. + /// + /// The type containing this operator. + /// + bool IsImplicitOperator(out ITypeSymbol containingType) + { + containingType = symbol.ContainingType; + if (containingType != null) + { + var containingNamedType = containingType as INamedTypeSymbol; + return containingNamedType == null || !containingNamedType.MemberNames.Contains(symbol.Name); + } + + var pointerType = symbol.Parameters.Select(p => p.Type).OfType().FirstOrDefault(); + if (pointerType != null) + { + containingType = pointerType; + return true; + } + + Context.ModelError(symbol, "Unexpected implicit operator"); + return true; + } + + /// + /// Convert an operator method name in to a symbolic name. + /// A return value indicates whether the conversion succeeded. + /// + /// The method name. + /// The converted operator name. + public static bool OperatorSymbol(string methodName, out string operatorName) + { + var success = true; + switch (methodName) + { + case "op_LogicalNot": + operatorName = "!"; + break; + case "op_BitwiseAnd": + operatorName = "&"; + break; + case "op_Equality": + operatorName = "=="; + break; + case "op_Inequality": + operatorName = "!="; + break; + case "op_UnaryPlus": + case "op_Addition": + operatorName = "+"; + break; + case "op_UnaryNegation": + case "op_Subtraction": + operatorName = "-"; + break; + case "op_Multiply": + operatorName = "*"; + break; + case "op_Division": + operatorName = "/"; + break; + case "op_Modulus": + operatorName = "%"; + break; + case "op_GreaterThan": + operatorName = ">"; + break; + case "op_GreaterThanOrEqual": + operatorName = ">="; + break; + case "op_LessThan": + operatorName = "<"; + break; + case "op_LessThanOrEqual": + operatorName = "<="; + break; + case "op_Decrement": + operatorName = "--"; + break; + case "op_Increment": + operatorName = "++"; + break; + case "op_Implicit": + operatorName = "implicit conversion"; + break; + case "op_Explicit": + operatorName = "explicit conversion"; + break; + case "op_OnesComplement": + operatorName = "~"; + break; + case "op_RightShift": + operatorName = ">>"; + break; + case "op_LeftShift": + operatorName = "<<"; + break; + case "op_BitwiseOr": + operatorName = "|"; + break; + case "op_ExclusiveOr": + operatorName = "^"; + break; + case "op_True": + operatorName = "true"; + break; + case "op_False": + operatorName = "false"; + break; + default: + operatorName = methodName; + success = false; + break; + } + return success; + } + + /// + /// Converts a method name into a symbolic name. + /// Logs an error if the name is not found. + /// + /// Extractor context. + /// The method name. + /// The converted name. + public static string OperatorSymbol(Context cx, string methodName) + { + string result; + if (!OperatorSymbol(methodName, out result)) + cx.ModelError("Unhandled operator name in OperatorSymbol(): '{0}'", methodName); + return result; + } + + public new static UserOperator Create(Context cx, IMethodSymbol symbol) => UserOperatorFactory.Instance.CreateEntity(cx, symbol); + + class UserOperatorFactory : ICachedEntityFactory + { + public static readonly UserOperatorFactory Instance = new UserOperatorFactory(); + + public UserOperator Create(Context cx, IMethodSymbol init) => new UserOperator(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs new file mode 100644 index 00000000000..ea1258ab59a --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs @@ -0,0 +1,61 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Entities; +using System.Collections.Generic; + +namespace Semmle.Extraction.CSharp.Entities +{ + class UsingDirective : FreshEntity + { + readonly UsingDirectiveSyntax node; + + public UsingDirective(Context cx, UsingDirectiveSyntax usingDirective, NamespaceDeclaration parent) + : base(cx) + { + node = usingDirective; + var info = cx.Model(node).GetSymbolInfo(usingDirective.Name); + + if (usingDirective.StaticKeyword.Kind() == SyntaxKind.None) + { + // A normal using + var namespaceSymbol = info.Symbol as INamespaceSymbol; + + if (namespaceSymbol == null) + { + cx.Extractor.MissingNamespace(usingDirective.Name.ToFullString()); + cx.ModelError(usingDirective, "Namespace not found"); + return; + } + else + { + var ns = Namespace.Create(cx, namespaceSymbol); + cx.Emit(Tuples.using_namespace_directives(this, ns)); + cx.Emit(Tuples.using_directive_location(this, cx.Create(ReportingLocation))); + } + } + else + { + // A "using static" + Type m = Type.Create(cx, (ITypeSymbol)info.Symbol); + cx.Emit(Tuples.using_static_directives(this, m.TypeRef)); + cx.Emit(Tuples.using_directive_location(this, cx.Create(ReportingLocation))); + } + + if (parent != null) + { + cx.Emit(Tuples.parent_namespace_declaration(this, parent)); + } + } + + public sealed override Microsoft.CodeAnalysis.Location ReportingLocation => node.GetLocation(); + + public IEnumerable GetTuples() + { + yield break; + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor.cs new file mode 100644 index 00000000000..10a145d9e2f --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor.cs @@ -0,0 +1,407 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.Text; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Semmle.Util; +using System.Text; +using System.Diagnostics; +using System.Threading.Tasks; +using Semmle.Util.Logging; +using System.Collections.Concurrent; + +namespace Semmle.Extraction.CSharp +{ + public static class Extractor + { + public enum ExitCode + { + Ok, // Everything worked perfectly + Errors, // Trap was generated but there were processing errors + Failed // Trap could not be generated + } + + class LogProgressMonitor : IProgressMonitor + { + readonly ILogger Logger; + + public LogProgressMonitor(ILogger logger) + { + Logger = logger; + } + + public void Analysed(int item, int total, string source, string output, TimeSpan time, AnalysisAction action) + { + Logger.Log(Severity.Info, " {0} -> {1} ({2})", source, output, + action == AnalysisAction.Extracted ? time.ToString() : action == AnalysisAction.Excluded ? "excluded" : "up to date"); + } + + public void MissingNamespace(string @namespace) { } + + public void MissingSummary(int types, int namespaces) { } + + public void MissingType(string type) { } + } + + /// + /// Command-line driver for the extractor. + /// + /// + /// + /// The extractor can be invoked in one of two ways: Either as an "analyser" passed in via the /a + /// option to csc.exe, or as a stand-alone executable. In this case, we need to faithfully + /// drive Roslyn in the way that csc.exe would. + /// + /// + /// Command line arguments as passed to csc.exe + /// + public static ExitCode Run(string[] args) + { + var commandLineArguments = Options.CreateWithEnvironment(args); + var fileLogger = new FileLogger(commandLineArguments.Verbosity, GetCSharpLogPath()); + var logger = commandLineArguments.Console + ? new CombinedLogger(new ConsoleLogger(commandLineArguments.Verbosity), fileLogger) + : (ILogger)fileLogger; + + if (Environment.GetEnvironmentVariable("SEMMLE_CLRTRACER") == "1" && !commandLineArguments.ClrTracer) + { + logger.Log(Severity.Info, "Skipping extraction since already extracted from the CLR tracer"); + return ExitCode.Ok; + } + + using (var analyser = new Analyser(new LogProgressMonitor(logger), logger)) + using (var references = new BlockingCollection()) + { + try + { + var compilerVersion = new CompilerVersion(commandLineArguments); + + bool preserveSymlinks = Environment.GetEnvironmentVariable("SEMMLE_PRESERVE_SYMLINKS") == "true"; + var canonicalPathCache = CanonicalPathCache.Create(logger, 1000, preserveSymlinks ? CanonicalPathCache.Symlinks.Preserve : CanonicalPathCache.Symlinks.Follow); + + if (compilerVersion.SkipExtraction) + { + logger.Log(Severity.Warning, " Unrecognized compiler '{0}' because {1}", compilerVersion.SpecifiedCompiler, compilerVersion.SkipReason); + return ExitCode.Ok; + } + + var argsWithResponse = AddDefaultResponse(compilerVersion.CscRsp, commandLineArguments.CompilerArguments); + + var cwd = Directory.GetCurrentDirectory(); + var compilerArguments = CSharpCommandLineParser.Default.Parse( + argsWithResponse, + cwd, + compilerVersion.FrameworkPath, + compilerVersion.AdditionalReferenceDirectories + ); + + if (compilerArguments == null) + { + var sb = new StringBuilder(); + sb.Append(" Failed to parse command line: ").AppendList(" ", args); + logger.Log(Severity.Error, sb.ToString()); + ++analyser.CompilationErrors; + return ExitCode.Failed; + } + + var referenceTasks = ResolveReferences(compilerArguments, analyser, canonicalPathCache, references); + + var syntaxTrees = new List(); + var syntaxTreeTasks = ReadSyntaxTrees( + compilerArguments.SourceFiles. + Select(src => canonicalPathCache.GetCanonicalPath(src.Path)), + analyser, + compilerArguments.ParseOptions, + compilerArguments.Encoding, + syntaxTrees); + + var sw = new Stopwatch(); + sw.Start(); + + Parallel.Invoke( + new ParallelOptions { MaxDegreeOfParallelism = commandLineArguments.Threads }, + referenceTasks.Interleave(syntaxTreeTasks).ToArray()); + + if (syntaxTrees.Count == 0) + { + logger.Log(Severity.Error, " No source files"); + ++analyser.CompilationErrors; + analyser.LogDiagnostics(); + return ExitCode.Failed; + } + + var compilation = CSharpCompilation.Create( + compilerArguments.CompilationName, + syntaxTrees, + references, + compilerArguments.CompilationOptions. + WithAssemblyIdentityComparer(DesktopAssemblyIdentityComparer.Default). + WithStrongNameProvider(new DesktopStrongNameProvider(compilerArguments.KeyFileSearchPaths)) + // csc.exe (CSharpCompiler.cs) also provides WithMetadataReferenceResolver, + // WithXmlReferenceResolver and + // WithSourceReferenceResolver. + // These would be needed if we hadn't explicitly provided the source/references + // already. + ); + + analyser.Initialize(compilerArguments, compilation, commandLineArguments); + analyser.AnalyseReferences(); + + foreach (var tree in compilation.SyntaxTrees) + { + analyser.AnalyseTree(tree); + } + + sw.Stop(); + logger.Log(Severity.Info, " Models constructed in {0}", sw.Elapsed); + + sw.Restart(); + analyser.PerformExtraction(commandLineArguments.Threads); + sw.Stop(); + logger.Log(Severity.Info, " Extraction took {0}", sw.Elapsed); + + return analyser.TotalErrors == 0 ? ExitCode.Ok : ExitCode.Errors; + } + catch (Exception e) + { + logger.Log(Severity.Error, " Unhandled exception: {0}", e); + return ExitCode.Errors; + } + } + } + + internal static bool SuppressDefaultResponseFile(IEnumerable args) + { + return args.Any(arg => new[] { "/noconfig", "-noconfig" }.Contains(arg.ToLowerInvariant())); + } + + /// + /// Adds @csc.rsp to the argument list to mimic csc.exe. + /// + /// The full pathname of csc.rsp. + /// The other command line arguments. + /// Modified list of arguments. + static IEnumerable AddDefaultResponse(string responseFile, IEnumerable args) + { + return SuppressDefaultResponseFile(args) && File.Exists(responseFile) ? + args : + new[] { "@" + responseFile }.Concat(args); + } + + /// + /// Gets the complete list of locations to locate references. + /// + /// Command line arguments. + /// List of directories. + static IEnumerable FixedReferencePaths(Microsoft.CodeAnalysis.CommandLineArguments args) + { + // See https://msdn.microsoft.com/en-us/library/s5bac5fx.aspx + // on how csc resolves references. Basically, + // 1) Current working directory. This is the directory from which the compiler is invoked. + // 2) The common language runtime system directory. + // 3) Directories specified by / lib. + // 4) Directories specified by the LIB environment variable. + + yield return args.BaseDirectory; + + foreach (var r in args.ReferencePaths) + yield return r; + + var lib = System.Environment.GetEnvironmentVariable("LIB"); + if (lib != null) + yield return lib; + } + + static MetadataReference MakeReference(CommandLineReference reference, string path) + { + return MetadataReference.CreateFromFile(path).WithProperties(reference.Properties); + } + + /// + /// Construct tasks for resolving references (possibly in parallel). + /// + /// The resolved references will be added (thread-safely) to the supplied + /// list . + /// + static IEnumerable ResolveReferences(Microsoft.CodeAnalysis.CommandLineArguments args, Analyser analyser, CanonicalPathCache canonicalPathCache, BlockingCollection ret) + { + var referencePaths = new Lazy(() => FixedReferencePaths(args).ToArray()); + return args.MetadataReferences.Select(clref => () => + { + if (Path.IsPathRooted(clref.Reference)) + { + if (File.Exists(clref.Reference)) + { + var reference = MakeReference(clref, canonicalPathCache.GetCanonicalPath(clref.Reference)); + ret.Add(reference); + } + else + { + lock (analyser) + { + analyser.Logger.Log(Severity.Error, " Reference '{0}' does not exist", clref.Reference); + ++analyser.CompilationErrors; + } + } + } + else + { + bool referenceFound = false; + { + foreach (var composed in referencePaths.Value. + Select(path => Path.Combine(path, clref.Reference)). + Where(path => File.Exists(path)). + Select(path => canonicalPathCache.GetCanonicalPath(path))) + { + referenceFound = true; + var reference = MakeReference(clref, composed); + ret.Add(reference); + break; + } + if (!referenceFound) + { + lock (analyser) + { + analyser.Logger.Log(Severity.Error, " Unable to resolve reference '{0}'", clref.Reference); + ++analyser.CompilationErrors; + } + } + } + } + }); + } + + /// + /// Construct tasks for reading source code files (possibly in parallel). + /// + /// The constructed syntax trees will be added (thread-safely) to the supplied + /// list . + /// + static IEnumerable ReadSyntaxTrees(IEnumerable sources, Analyser analyser, CSharpParseOptions parseOptions, Encoding encoding, IList ret) + { + return sources.Select(path => () => + { + try + { + using (var file = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + var st = CSharpSyntaxTree.ParseText(SourceText.From(file, encoding), parseOptions, path); + lock (ret) + ret.Add(st); + } + } + catch (IOException ex) + { + lock (analyser) + { + analyser.Logger.Log(Severity.Error, " Unable to open source file {0}: {1}", path, ex.Message); + ++analyser.CompilationErrors; + } + } + }); + } + + public static void ExtractStandalone( + IEnumerable sources, + IEnumerable referencePaths, + IProgressMonitor pm, + ILogger logger, + CommonOptions options) + { + using (var analyser = new Analyser(pm, logger)) + using (var references = new BlockingCollection()) + { + try + { + var referenceTasks = referencePaths.Select(path => () => + { + var reference = MetadataReference.CreateFromFile(path); + references.Add(reference); + }); + + var syntaxTrees = new List(); + var syntaxTreeTasks = ReadSyntaxTrees(sources, analyser, null, null, syntaxTrees); + + var sw = new Stopwatch(); + sw.Start(); + + Parallel.Invoke( + new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, + referenceTasks.Interleave(syntaxTreeTasks).ToArray()); + + if (syntaxTrees.Count == 0) + { + analyser.Logger.Log(Severity.Error, " No source files"); + ++analyser.CompilationErrors; + } + + var compilation = CSharpCompilation.Create( + "csharp.dll", + syntaxTrees, + references + ); + + analyser.InitializeStandalone(compilation, options); + analyser.AnalyseReferences(); + + foreach (var tree in compilation.SyntaxTrees) + { + analyser.AnalyseTree(tree); + } + + sw.Stop(); + analyser.Logger.Log(Severity.Info, " Models constructed in {0}", sw.Elapsed); + + sw.Restart(); + analyser.PerformExtraction(options.Threads); + sw.Stop(); + analyser.Logger.Log(Severity.Info, " Extraction took {0}", sw.Elapsed); + + foreach (var type in analyser.MissingNamespaces) + { + pm.MissingNamespace(type); + } + + foreach (var type in analyser.MissingTypes) + { + pm.MissingType(type); + } + + pm.MissingSummary(analyser.MissingTypes.Count(), analyser.MissingNamespaces.Count()); + } + catch (Exception e) + { + analyser.Logger.Log(Severity.Error, " Unhandled exception: {0}", e); + } + } + } + + /// + /// Gets the path to the `csharp.log` file written to by the C# extractor. + /// + public static string GetCSharpLogPath() + { + string snapshot = Environment.GetEnvironmentVariable("ODASA_SNAPSHOT"); + string buildErrorDir = Environment.GetEnvironmentVariable("ODASA_BUILD_ERROR_DIR"); + string traps = Environment.GetEnvironmentVariable("TRAP_FOLDER"); + string output = "csharp.log"; + if (!string.IsNullOrEmpty(snapshot)) + { + snapshot = Path.Combine(snapshot, "log"); + return Path.Combine(snapshot, output); + } + if (!string.IsNullOrEmpty(buildErrorDir)) + { + // Used by `qltest` + return Path.Combine(buildErrorDir, output); + } + if (!string.IsNullOrEmpty(traps)) + { + return Path.Combine(traps, output); + } + return output; + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Kinds/ExprKind.cs b/csharp/extractor/Semmle.Extraction.CSharp/Kinds/ExprKind.cs new file mode 100644 index 00000000000..daa05a494ae --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Kinds/ExprKind.cs @@ -0,0 +1,110 @@ +namespace Semmle.Extraction.Kinds +{ + /// + /// This enum has been auto-generated from the C# DB scheme - do not edit. + /// Auto-generate command: `genkindenum.pl expr` + /// + public enum ExprKind + { + BOOL_LITERAL = 1, + CHAR_LITERAL = 2, + DECIMAL_LITERAL = 3, + INT_LITERAL = 4, + LONG_LITERAL = 5, + UINT_LITERAL = 6, + ULONG_LITERAL = 7, + FLOAT_LITERAL = 8, + DOUBLE_LITERAL = 9, + STRING_LITERAL = 10, + NULL_LITERAL = 11, + THIS_ACCESS = 12, + BASE_ACCESS = 13, + LOCAL_VARIABLE_ACCESS = 14, + PARAMETER_ACCESS = 15, + FIELD_ACCESS = 16, + PROPERTY_ACCESS = 17, + METHOD_ACCESS = 18, + EVENT_ACCESS = 19, + INDEXER_ACCESS = 20, + ARRAY_ACCESS = 21, + TYPE_ACCESS = 22, + TYPEOF = 23, + METHOD_INVOCATION = 24, + DELEGATE_INVOCATION = 25, + OPERATOR_INVOCATION = 26, + CAST = 27, + OBJECT_CREATION = 28, + EXPLICIT_DELEGATE_CREATION = 29, + IMPLICIT_DELEGATE_CREATION = 30, + ARRAY_CREATION = 31, + DEFAULT = 32, + PLUS = 33, + MINUS = 34, + BIT_NOT = 35, + LOG_NOT = 36, + POST_INCR = 37, + POST_DECR = 38, + PRE_INCR = 39, + PRE_DECR = 40, + MUL = 41, + DIV = 42, + REM = 43, + ADD = 44, + SUB = 45, + LSHIFT = 46, + RSHIFT = 47, + LT = 48, + GT = 49, + LE = 50, + GE = 51, + EQ = 52, + NE = 53, + BIT_AND = 54, + BIT_XOR = 55, + BIT_OR = 56, + LOG_AND = 57, + LOG_OR = 58, + IS = 59, + AS = 60, + NULL_COALESCING = 61, + CONDITIONAL = 62, + SIMPLE_ASSIGN = 63, + ASSIGN_ADD = 64, + ASSIGN_SUB = 65, + ASSIGN_MUL = 66, + ASSIGN_DIV = 67, + ASSIGN_REM = 68, + ASSIGN_AND = 69, + ASSIGN_XOR = 70, + ASSIGN_OR = 71, + ASSIGN_LSHIFT = 72, + ASSIGN_RSHIFT = 73, + OBJECT_INIT = 74, + COLLECTION_INIT = 75, + ARRAY_INIT = 76, + CHECKED = 77, + UNCHECKED = 78, + CONSTRUCTOR_INIT = 79, + ADD_EVENT = 80, + REMOVE_EVENT = 81, + PAR = 82, + LOCAL_VAR_DECL = 83, + LAMBDA = 84, + ANONYMOUS_METHOD = 85, + NAMESPACE = 86, + DYNAMIC_ELEMENT_ACCESS = 92, + DYNAMIC_MEMBER_ACCESS = 93, + POINTER_INDIRECTION = 100, + ADDRESS_OF = 101, + SIZEOF = 102, + AWAIT = 103, + NAMEOF = 104, + INTERPOLATED_STRING = 105, + UNKNOWN = 106, + THROW = 107, + TUPLE = 108, + LOCAL_FUNCTION_INVOCATION = 109, + REF = 110, + DISCARD = 111 + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Kinds/StmtKind.cs b/csharp/extractor/Semmle.Extraction.CSharp/Kinds/StmtKind.cs new file mode 100644 index 00000000000..1ecf005a360 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Kinds/StmtKind.cs @@ -0,0 +1,39 @@ +namespace Semmle.Extraction.Kinds +{ + /// + /// This enum has been auto-generated from the C# DB scheme - do not edit. + /// + public enum StmtKind + { + BLOCK = 1, + EXPR = 2, + IF = 3, + SWITCH = 4, + WHILE = 5, + DO = 6, + FOR = 7, + FOREACH = 8, + BREAK = 9, + CONTINUE = 10, + GOTO = 11, + GOTO_CASE = 12, + GOTO_DEFAULT = 13, + THROW = 14, + RETURN = 15, + YIELD = 16, + TRY = 17, + CHECKED = 18, + UNCHECKED = 19, + LOCK = 20, + USING = 21, + VAR_DECL = 22, + CONST_DECL = 23, + EMPTY = 24, + UNSAFE = 25, + FIXED = 26, + LABEL = 27, + CATCH = 28, + CASE = 29, + LOCAL_FUNCTION = 30 + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Kinds/TypeKind.cs b/csharp/extractor/Semmle.Extraction.CSharp/Kinds/TypeKind.cs new file mode 100644 index 00000000000..6387193db3e --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Kinds/TypeKind.cs @@ -0,0 +1,39 @@ +namespace Semmle.Extraction.Kinds +{ + /// + /// This enum has been auto-generated from the C# DB scheme - do not edit. + /// + public enum TypeKind + { + BOOL = 1, + CHAR = 2, + DECIMAL = 3, + SBYTE = 4, + SHORT = 5, + INT = 6, + LONG = 7, + BYTE = 8, + USHORT = 9, + UINT = 10, + ULONG = 11, + FLOAT = 12, + DOUBLE = 13, + ENUM = 14, + STRUCT = 15, + CLASS = 17, + INTERFACE = 19, + DELEGATE = 20, + NULL = 21, + TYPE_PARAMETER = 22, + POINTER = 23, + NULLABLE = 24, + ARRAY = 25, + VOID = 26, + INT_PTR = 27, + UINT_PTR = 28, + DYNAMIC = 29, + ARGLIST = 30, + UNKNOWN = 31, + TUPLE = 32 + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Options.cs b/csharp/extractor/Semmle.Extraction.CSharp/Options.cs new file mode 100644 index 00000000000..8fa7be08b67 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Options.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using Semmle.Util; + +namespace Semmle.Extraction.CSharp +{ + public sealed class Options : CommonOptions + { + /// + /// The compiler exe, or null if unspecified. + /// + public string CompilerName; + + /// + /// Specified .Net Framework dir, or null if unspecified. + /// + public string Framework; + + /// + /// All other arguments passed to the compilation. + /// + public IList CompilerArguments = new List(); + + /// + /// Holds if the extractor was launched from the CLR tracer. + /// + public bool ClrTracer = false; + + public static Options CreateWithEnvironment(string[] arguments) + { + var options = new Options(); + var extractionOptions = Environment.GetEnvironmentVariable("SEMMLE_EXTRACTOR_OPTIONS") ?? + Environment.GetEnvironmentVariable("LGTM_INDEX_EXTRACTOR"); + + var argsList = new List(arguments); + + if (!string.IsNullOrEmpty(extractionOptions)) + argsList.AddRange(extractionOptions.Split(' ')); + + options.ParseArguments(argsList); + return options; + } + + public override bool handleArgument(string argument) + { + CompilerArguments.Add(argument); + return true; + } + + public override void invalidArgument(string argument) + { + // Unrecognised arguments are passed to the compiler. + CompilerArguments.Add(argument); + } + + public override bool handleOption(string key, string value) + { + switch (key) + { + case "compiler": + CompilerName = value; + return true; + case "framework": + Framework = value; + return true; + default: + return base.handleOption(key, value); + } + } + + public override bool handleFlag(string flag, bool value) + { + switch (flag) + { + case "clrtracer": + ClrTracer = value; + return true; + default: + return base.handleFlag(flag, value); + } + } + + private Options() + { + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Populators/Ast.cs b/csharp/extractor/Semmle.Extraction.CSharp/Populators/Ast.cs new file mode 100644 index 00000000000..0ee8117ff8c --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Populators/Ast.cs @@ -0,0 +1,69 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Entities; + +namespace Semmle.Extraction.CSharp.Populators +{ + class Ast : CSharpSyntaxVisitor + { + readonly Context cx; + readonly IExpressionParentEntity parent; + readonly int child; + + public Ast(Context cx, IExpressionParentEntity parent, int child) + { + this.cx = cx; + this.parent = parent; + this.child = child; + } + + public override void DefaultVisit(SyntaxNode node) + { + cx.ModelError(node, "Unhandled syntax node {0}", node.Kind()); + } + + public override void VisitPropertyDeclaration(PropertyDeclarationSyntax node) + { + ((Property)parent).VisitDeclaration(cx, node); + } + + + public override void VisitArgumentList(ArgumentListSyntax node) + { + int c = 0; + foreach (var m in node.Arguments) + { + cx.Extract(m, parent, c++); + } + } + + public override void VisitArgument(ArgumentSyntax node) + { + Expression.Create(cx, node.Expression, parent, child); + } + } + + public static class AstExtensions + { + public static void Extract(this Context cx, CSharpSyntaxNode node, IExpressionParentEntity parent, int child) + { + using (cx.StackGuard) + { + try + { + node.Accept(new Ast(cx, parent, child)); + } + catch (System.Exception e) + { + cx.ModelError(node, "Exception processing syntax node of type {0}: {1}", node.Kind(), e); + } + } + } + + public static void Extract(this Context cx, SyntaxNode node, IEntity parent, int child) + { + cx.Extract(((CSharpSyntaxNode)node), parent, child); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Populators/Comments.cs b/csharp/extractor/Semmle.Extraction.CSharp/Populators/Comments.cs new file mode 100644 index 00000000000..8f6bb6f5206 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Populators/Comments.cs @@ -0,0 +1,33 @@ +using Semmle.Extraction.CommentProcessing; +using System; + +namespace Semmle.Extraction.CSharp.Populators +{ + /// + /// Populators for comments. + /// + public static class Comments + { + public static void ExtractComments(this Context cx, ICommentGenerator gen) + { + cx.Try(null, null, () => + { + gen.GenerateBindings((entity, duplicationGuardKey, block, binding) => + { + var commentBlock = Entities.CommentBlock.Create(cx, block); + Action a = () => + { + commentBlock.BindTo(entity, binding); + }; + // When the duplication guard key exists, it means that the entity is guarded against + // trap duplication (). + // We must therefore also guard comment construction. + if (duplicationGuardKey != null) + cx.WithDuplicationGuard(duplicationGuardKey, a); + else + a(); + }); + }); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Populators/CompilationUnit.cs b/csharp/extractor/Semmle.Extraction.CSharp/Populators/CompilationUnit.cs new file mode 100644 index 00000000000..dd87f483ddf --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Populators/CompilationUnit.cs @@ -0,0 +1,125 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Entities; +using Semmle.Extraction.Entities; +using Semmle.Util.Logging; + +namespace Semmle.Extraction.CSharp.Populators +{ + public class TypeContainerVisitor : CSharpSyntaxVisitor + { + protected readonly Context cx; + protected readonly IEntity parent; + + public TypeContainerVisitor(Context cx, IEntity parent) + { + this.cx = cx; + this.parent = parent; + } + + public override void DefaultVisit(SyntaxNode node) + { + throw new InternalError(node, "Unhandled top-level syntax node"); + } + + public override void VisitDelegateDeclaration(DelegateDeclarationSyntax node) + { + Entities.NamedType.Create(cx, cx.Model(node).GetDeclaredSymbol(node)).ExtractRecursive(parent); + } + + public override void VisitClassDeclaration(ClassDeclarationSyntax classDecl) + { + Entities.Type.Create(cx, cx.Model(classDecl).GetDeclaredSymbol(classDecl)).ExtractRecursive(parent); + } + + public override void VisitStructDeclaration(StructDeclarationSyntax node) + { + Entities.Type.Create(cx, cx.Model(node).GetDeclaredSymbol(node)).ExtractRecursive(parent); + } + + public override void VisitEnumDeclaration(EnumDeclarationSyntax node) + { + Entities.Type.Create(cx, cx.Model(node).GetDeclaredSymbol(node)).ExtractRecursive(parent); + } + + public override void VisitInterfaceDeclaration(InterfaceDeclarationSyntax node) + { + Entities.Type.Create(cx, cx.Model(node).GetDeclaredSymbol(node)).ExtractRecursive(parent); + } + + public override void VisitAttributeList(AttributeListSyntax node) + { + if (cx.Extractor.Standalone) return; + + var outputAssembly = Assembly.CreateOutputAssembly(cx); + foreach (var attribute in node.Attributes) + { + var ae = new Attribute(cx, attribute, outputAssembly); + cx.BindComments(ae, attribute.GetLocation()); + } + } + } + + class TypeOrNamespaceVisitor : TypeContainerVisitor + { + public TypeOrNamespaceVisitor(Context cx, IEntity parent) + : base(cx, parent) { } + + public override void VisitUsingDirective(UsingDirectiveSyntax usingDirective) + { + // Only deal with "using namespace" not "using X = Y" + if (usingDirective.Alias == null) + new UsingDirective(cx, usingDirective, (NamespaceDeclaration)parent); + } + + public override void VisitNamespaceDeclaration(NamespaceDeclarationSyntax node) + { + NamespaceDeclaration.Create(cx, node, (NamespaceDeclaration)parent); + } + } + + class CompilationUnitVisitor : TypeOrNamespaceVisitor + { + public CompilationUnitVisitor(Context cx) + : base(cx, null) { } + + public override void VisitExternAliasDirective(ExternAliasDirectiveSyntax node) + { + // This information is not yet extracted. + cx.Extractor.Message(new Message { severity = Severity.Info, message = "Ignoring extern alias directive" }); + } + + public override void VisitCompilationUnit(CompilationUnitSyntax compilationUnit) + { + foreach (var m in compilationUnit.ChildNodes()) + { + cx.Try(m, null, () => ((CSharpSyntaxNode)m).Accept(this)); + } + + // Gather comments: + foreach (SyntaxTrivia trivia in compilationUnit.DescendantTrivia(compilationUnit.Span)) + { + CommentLine.Extract(cx, trivia); + } + + foreach (var trivia in compilationUnit.GetLeadingTrivia()) + { + CommentLine.Extract(cx, trivia); + } + + foreach (var trivia in compilationUnit.GetTrailingTrivia()) + { + CommentLine.Extract(cx, trivia); + } + } + } + + public class CompilationUnit + { + public static void Extract(Context cx, SyntaxNode unit) + { + ((CSharpSyntaxNode)unit).Accept(new CompilationUnitVisitor(cx)); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Populators/Locations.cs b/csharp/extractor/Semmle.Extraction.CSharp/Populators/Locations.cs new file mode 100644 index 00000000000..a9a4863a2e4 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Populators/Locations.cs @@ -0,0 +1,104 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Populators +{ + public static class LocationExtensions + { + /// + /// Manually extend a location. + /// + /// The location to extend. + /// The node to extend the location to. + /// Extended location. + public static Location ExtendLocation(this Location l1, SyntaxNode n2) + { + if (n2 == null) + { + return l1; + } + else + { + var l2 = n2.FixedLocation(); + int start = System.Math.Min(l1.SourceSpan.Start, l2.SourceSpan.Start); + int end = System.Math.Max(l1.SourceSpan.End, l2.SourceSpan.End); + return Location.Create(n2.SyntaxTree, new Microsoft.CodeAnalysis.Text.TextSpan(start, end - start)); + } + } + + /// + /// Adjust the location of some syntax nodes + /// to make them more suitable for displaying results. + /// Sometimes we do not wish to highlight the whole node, + /// so select a sub-node such as the name. + /// !! Refactor this into each entity. + /// + /// The syntax node. + /// The fixed location. + public static Location FixedLocation(this SyntaxNode node) + { + Location result; + switch (node.Kind()) + { + case SyntaxKind.EqualsValueClause: + result = ((EqualsValueClauseSyntax)node).Value.FixedLocation(); + break; + case SyntaxKind.OperatorDeclaration: + { + var decl = (OperatorDeclarationSyntax)node; + result = decl.OperatorKeyword.GetLocation().ExtendLocation(decl.ParameterList); + break; + } + case SyntaxKind.ConversionOperatorDeclaration: + { + var decl = (ConversionOperatorDeclarationSyntax)node; + result = decl.OperatorKeyword.GetLocation(); + break; + } + case SyntaxKind.DelegateDeclaration: + { + var decl = (DelegateDeclarationSyntax)node; + return decl.Identifier.GetLocation().ExtendLocation(decl.TypeParameterList); + } + case SyntaxKind.ClassDeclaration: + case SyntaxKind.StructDeclaration: + case SyntaxKind.InterfaceDeclaration: + { + var decl = (TypeDeclarationSyntax)node; + return decl.Identifier.GetLocation().ExtendLocation(decl.TypeParameterList); + } + case SyntaxKind.EnumDeclaration: + return ((EnumDeclarationSyntax)node).Identifier.GetLocation(); + case SyntaxKind.MethodDeclaration: + { + var decl = (MethodDeclarationSyntax)node; + return decl.Identifier.GetLocation().ExtendLocation(decl.TypeParameterList); + } + case SyntaxKind.ConstructorDeclaration: + { + var decl = (ConstructorDeclarationSyntax)node; + return decl.Identifier.GetLocation(); + } + case SyntaxKind.ParenthesizedExpression: + return ((ParenthesizedExpressionSyntax)node).Expression.FixedLocation(); + case SyntaxKind.CatchDeclaration: + return ((CatchDeclarationSyntax)node).Identifier.GetLocation(); + case SyntaxKind.LabeledStatement: + return ((LabeledStatementSyntax)node).Identifier.GetLocation(); + default: + result = node.GetLocation(); + break; + } + return result; + } + + public static Location GetSymbolLocation(this ISymbol symbol) + { + return symbol.DeclaringSyntaxReferences.Any() ? + symbol.DeclaringSyntaxReferences.First().GetSyntax().FixedLocation() : + symbol.Locations.First(); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Populators/Methods.cs b/csharp/extractor/Semmle.Extraction.CSharp/Populators/Methods.cs new file mode 100644 index 00000000000..109d07f6f7d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Populators/Methods.cs @@ -0,0 +1,64 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Util; + +namespace Semmle.Extraction.CSharp.Populators +{ + public static class MethodExtensions + { + class AstLineCounter : CSharpSyntaxVisitor + { + public override LineCounts DefaultVisit(SyntaxNode node) + { + string text = node.SyntaxTree.GetText().GetSubText(node.GetLocation().SourceSpan).ToString(); + return Semmle.Util.LineCounter.ComputeLineCounts(text); + } + + public override LineCounts VisitMethodDeclaration(MethodDeclarationSyntax method) + { + return Visit(method.Identifier, method.Body ?? (SyntaxNode)method.ExpressionBody); + } + + public LineCounts Visit(SyntaxToken identifier, SyntaxNode body) + { + int start = identifier.GetLocation().SourceSpan.Start; + int end = body.GetLocation().SourceSpan.End - 1; + + var textSpan = new Microsoft.CodeAnalysis.Text.TextSpan(start, end - start); + + string text = body.SyntaxTree.GetText().GetSubText(textSpan) + "\r\n"; + return Semmle.Util.LineCounter.ComputeLineCounts(text); + } + + public override LineCounts VisitConstructorDeclaration(ConstructorDeclarationSyntax method) + { + return Visit(method.Identifier, (SyntaxNode)method.Body ?? method.ExpressionBody); + } + + public override LineCounts VisitDestructorDeclaration(DestructorDeclarationSyntax method) + { + return Visit(method.Identifier, (SyntaxNode)method.Body ?? method.ExpressionBody); + } + + public override LineCounts VisitOperatorDeclaration(OperatorDeclarationSyntax node) + { + return Visit(node.OperatorToken, node.Body ?? (SyntaxNode)node.ExpressionBody); + } + } + + public static void NumberOfLines(this Context cx, ISymbol symbol, IEntity callable) + { + foreach (var decl in symbol.DeclaringSyntaxReferences) + { + cx.NumberOfLines((CSharpSyntaxNode)decl.GetSyntax(), callable); + } + } + + public static void NumberOfLines(this Context cx, CSharpSyntaxNode node, IEntity callable) + { + var lineCounts = node.Accept(new AstLineCounter()); + cx.Emit(Tuples.numlines(callable, lineCounts)); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Populators/Symbols.cs b/csharp/extractor/Semmle.Extraction.CSharp/Populators/Symbols.cs new file mode 100644 index 00000000000..0a62e4beaf0 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Populators/Symbols.cs @@ -0,0 +1,136 @@ +using System; +using System.Linq; +using Microsoft.CodeAnalysis; +using Semmle.Extraction.CSharp.Entities; + +namespace Semmle.Extraction.CSharp.Populators +{ + class Symbols : SymbolVisitor + { + readonly Context cx; + + public Symbols(Context cx) + { + this.cx = cx; + } + + public override IEntity DefaultVisit(ISymbol symbol) => throw new InternalError(symbol, "Unhandled symbol '{0}' of kind '{1}'", symbol, symbol.Kind); + + public override IEntity VisitArrayType(IArrayTypeSymbol array) => ArrayType.Create(cx, array); + + public override IEntity VisitMethod(IMethodSymbol methodDecl) + { + return Method.Create(cx, methodDecl); + } + + public override IEntity VisitField(IFieldSymbol field) => Field.Create(cx, field); + + public override IEntity VisitNamedType(INamedTypeSymbol type) => + type.IsTupleType ? TupleType.Create(cx, type) : (IEntity)NamedType.Create(cx, type); + + public override IEntity VisitNamespace(INamespaceSymbol ns) => Namespace.Create(cx, ns); + + public override IEntity VisitParameter(IParameterSymbol param) => Parameter.GetAlreadyCreated(cx, param); + + public override IEntity VisitProperty(IPropertySymbol symbol) => Property.Create(cx, symbol); + + public override IEntity VisitEvent(IEventSymbol symbol) => Event.Create(cx, symbol); + + public override IEntity VisitTypeParameter(ITypeParameterSymbol param) => TypeParameter.Create(cx, param); + + public override IEntity VisitPointerType(IPointerTypeSymbol symbol) => PointerType.Create(cx, symbol); + + public override IEntity VisitDynamicType(IDynamicTypeSymbol symbol) => DynamicType.Create(cx, symbol); + } + + public static class SymbolsExtensions + { + public static IEntity CreateEntity(this Context cx, ISymbol symbol) + { + if (symbol == null) return null; + + using (cx.StackGuard) + { + try + { + return symbol.Accept(new Symbols(cx)); + } + catch (Exception e) + { + cx.ModelError(symbol, "Exception processing symbol '{2}' of type '{0}': {1}", symbol.Kind, e, symbol); + return null; + } + } + } + + /// + /// Tries to recover from an ErrorType. + /// + /// + /// Extraction context. + /// The type to disambiguate. + /// + public static ITypeSymbol DisambiguateType(this Context cx, ITypeSymbol type) + { + /* A type could not be determined. + * Sometimes this happens due to a missing reference, + * or sometimes because the same type is defined in multiple places. + * + * In the case that a symbol is multiply-defined, Roslyn tells you which + * symbols are candidates. It usually resolves to the same DB entity, + * so it's reasonably safe to just pick a candidate. + * + * The conservative option would be to resolve all error types as null. + */ + + var errorType = type as IErrorTypeSymbol; + + return errorType != null && errorType.CandidateSymbols.Any() ? + errorType.CandidateSymbols.First() as ITypeSymbol : + type; + } + + public static TypeInfo GetTypeInfo(this Context cx, Microsoft.CodeAnalysis.CSharp.CSharpSyntaxNode node) => + cx.Model(node).GetTypeInfo(node); + + public static SymbolInfo GetSymbolInfo(this Context cx, Microsoft.CodeAnalysis.CSharp.CSharpSyntaxNode node) => + cx.Model(node).GetSymbolInfo(node); + + /// + /// Gets the symbol for a particular syntax node. + /// Throws an exception if the symbol is not found. + /// + /// + /// + /// This gives a nicer message than a "null pointer exception", + /// and should be used where we require a symbol to be resolved. + /// + /// + /// The extraction context. + /// The syntax node. + /// The resolved symbol. + public static ISymbol GetSymbol(this Context cx, Microsoft.CodeAnalysis.CSharp.CSharpSyntaxNode node) + { + var info = GetSymbolInfo(cx, node); + if (info.Symbol == null) + { + throw new InternalError(node, "Could not resolve symbol"); + } + + return info.Symbol; + } + + /// + /// Determines the type of a node, or null + /// if the type could not be determined. + /// + /// Extractor context. + /// The node to determine. + /// The type symbol of the node, or null. + public static ITypeSymbol GetType(this Context cx, Microsoft.CodeAnalysis.CSharp.CSharpSyntaxNode node) + { + var info = GetTypeInfo(cx, node); + return cx.DisambiguateType(info.Type); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Properties/AssemblyInfo.cs b/csharp/extractor/Semmle.Extraction.CSharp/Properties/AssemblyInfo.cs new file mode 100644 index 00000000000..bb2ea1f4882 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Properties/AssemblyInfo.cs @@ -0,0 +1,35 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Semmle.Extraction.CSharp")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Semmle.Extraction.CSharp")] +[assembly: AssemblyCopyright("Copyright © 2015")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("09d7c55d-5ed3-4af8-9b9f-8f9342533ee9")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Semmle.Extraction.CSharp.csproj b/csharp/extractor/Semmle.Extraction.CSharp/Semmle.Extraction.CSharp.csproj new file mode 100644 index 00000000000..9cf59fd7a28 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Semmle.Extraction.CSharp.csproj @@ -0,0 +1,25 @@ + + + + netcoreapp2.0 + Semmle.Extraction.CSharp + Semmle.Extraction.CSharp + false + true + + + + + + + + + + + + + + + + + diff --git a/csharp/extractor/Semmle.Extraction.CSharp/SymbolExtensions.cs b/csharp/extractor/Semmle.Extraction.CSharp/SymbolExtensions.cs new file mode 100644 index 00000000000..0a2dacdec1c --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/SymbolExtensions.cs @@ -0,0 +1,371 @@ +using Microsoft.CodeAnalysis; +using Semmle.Extraction.CSharp.Entities; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Semmle.Extraction.CSharp +{ + static class SymbolExtensions + { + /// + /// Gets the name of this symbol. + /// + /// If the symbol implements an explicit interface, only the + /// name of the member being implemented is included, not the + /// explicit prefix. + /// + public static string GetName(this ISymbol symbol, bool useMetadataName = false) + { + var name = useMetadataName ? symbol.MetadataName : symbol.Name; + return symbol.CanBeReferencedByName ? name : name.Substring(symbol.Name.LastIndexOf('.') + 1); + } + + /// + /// Gets the source-level modifiers belonging to this symbol, if any. + /// + public static IEnumerable GetSourceLevelModifiers(this ISymbol symbol) + { + var methodModifiers = + symbol.DeclaringSyntaxReferences. + Select(r => r.GetSyntax()). + OfType(). + SelectMany(md => md.Modifiers); + var typeModifers = + symbol.DeclaringSyntaxReferences. + Select(r => r.GetSyntax()). + OfType(). + SelectMany(cd => cd.Modifiers); + return methodModifiers.Concat(typeModifers).Select(m => m.Text); + } + + /// + /// Holds if this type symbol contains a type parameter from the + /// declaring generic . + /// + public static bool ContainsTypeParameters(this ITypeSymbol type, Context cx, ISymbol declaringGeneric) + { + using (cx.StackGuard) + { + switch (type.TypeKind) + { + case TypeKind.Array: + var array = (IArrayTypeSymbol)type; + return array.ElementType.ContainsTypeParameters(cx, declaringGeneric); + case TypeKind.Class: + case TypeKind.Interface: + case TypeKind.Struct: + case TypeKind.Enum: + case TypeKind.Delegate: + case TypeKind.Error: + var named = (INamedTypeSymbol)type; + if (named.IsTupleType) + named = named.TupleUnderlyingType; + if (named.ContainingType != null && named.ContainingType.ContainsTypeParameters(cx, declaringGeneric)) + return true; + return named.TypeArguments.Any(arg => arg.ContainsTypeParameters(cx, declaringGeneric)); + case TypeKind.Pointer: + var ptr = (IPointerTypeSymbol)type; + return ptr.PointedAtType.ContainsTypeParameters(cx, declaringGeneric); + case TypeKind.TypeParameter: + var tp = (ITypeParameterSymbol)type; + var declaringGen = tp.TypeParameterKind == TypeParameterKind.Method ? tp.DeclaringMethod : (ISymbol)tp.DeclaringType; + return Equals(declaringGen, declaringGeneric); + default: + return false; + } + } + } + + /// + /// Constructs a unique string for this type symbol. + /// + /// The supplied action is applied to the + /// syntactic sub terms of this type (if any). + /// + /// The extraction context. + /// The trap builder used to store the result. + /// The action to apply to syntactic sub terms of this type. + public static void BuildTypeId(this ITypeSymbol type, Context cx, ITrapBuilder tb, Action subTermAction) + { + if (type.SpecialType != SpecialType.None) + { + /* + * Use the keyword ("int" etc) for the built-in types. + * This makes the IDs shorter and means that all built-in types map to + * the same entities (even when using multiple versions of mscorlib). + */ + tb.Append(type.ToDisplayString()); + return; + } + + using (cx.StackGuard) + { + switch (type.TypeKind) + { + case TypeKind.Array: + var array = (IArrayTypeSymbol)type; + subTermAction(cx, tb, array.ElementType); + array.BuildArraySuffix(tb); + return; + case TypeKind.Class: + case TypeKind.Interface: + case TypeKind.Struct: + case TypeKind.Enum: + case TypeKind.Delegate: + case TypeKind.Error: + var named = (INamedTypeSymbol)type; + named.BuildNamedTypeId(cx, tb, subTermAction); + return; + case TypeKind.Pointer: + var ptr = (IPointerTypeSymbol)type; + subTermAction(cx, tb, ptr.PointedAtType); + tb.Append("*"); + return; + case TypeKind.TypeParameter: + var tp = (ITypeParameterSymbol)type; + tb.Append(tp.Name); + return; + case TypeKind.Dynamic: + tb.Append("dynamic"); + return; + default: + throw new InternalError(type, "Unhandled type kind '{0}'", type.TypeKind); + } + } + } + + /// + /// Constructs an array suffix string for this array type symbol. + /// + /// The trap builder used to store the result. + public static void BuildArraySuffix(this IArrayTypeSymbol array, ITrapBuilder tb) + { + tb.Append("["); + for (int i = 0; i < array.Rank - 1; i++) + tb.Append(","); + tb.Append("]"); + } + + static void BuildNamedTypeId(this INamedTypeSymbol named, Context cx, ITrapBuilder tb, Action subTermAction) + { + if (named.IsTupleType) + { + tb.Append("("); + tb.BuildList(",", named.TupleElements, + (f, tb0) => + { + tb.Append(f.Name).Append(":"); + subTermAction(cx, tb0, f.Type); + } + ); + tb.Append(")"); + return; + } + + if (named.ContainingType != null) + { + subTermAction(cx, tb, named.ContainingType); + tb.Append("."); + } + else if (named.ContainingNamespace != null) + { + named.ContainingNamespace.BuildNamespace(cx, tb); + } + + if (named.IsAnonymousType) + named.BuildAnonymousName(cx, tb, subTermAction, true); + else if (named.TypeParameters.IsEmpty) + tb.Append(named.Name); + else if (IsReallyUnbound(named)) + tb.Append(named.Name).Append("`").Append(named.TypeParameters.Length); + else + { + subTermAction(cx, tb, named.ConstructedFrom); + tb.Append("<"); + tb.BuildList(",", named.TypeArguments, (ta, tb0) => subTermAction(cx, tb0, ta)); + tb.Append(">"); + } + } + + static void BuildNamespace(this INamespaceSymbol ns, Context cx, ITrapBuilder tb) + { + // Only include the assembly information in each type ID + // for normal extractions. This is because standalone extractions + // lack assembly information or may be ambiguous. + bool prependAssemblyToTypeId = !cx.Extractor.Standalone && ns.ContainingAssembly != null; + + if (prependAssemblyToTypeId) + { + // Note that we exclude the revision number as this has + // been observed to be unstable. + var assembly = ns.ContainingAssembly.Identity; + tb.Append(assembly.Name).Append("_"). + Append(assembly.Version.Major).Append("."). + Append(assembly.Version.Minor).Append("."). + Append(assembly.Version.Build).Append("::"); + } + + tb.Append(Namespace.Create(cx, ns)).Append("."); + } + + static void BuildAnonymousName(this ITypeSymbol type, Context cx, ITrapBuilder tb, Action subTermAction, bool includeParamName) + { + var buildParam = includeParamName + ? (prop, tb0) => + { + tb0.Append(prop.Name).Append(" "); + subTermAction(cx, tb0, prop.Type); + } + : (Action)((prop, tb0) => subTermAction(cx, tb0, prop.Type)); + int memberCount = type.GetMembers().OfType().Count(); + int hackTypeNumber = memberCount == 1 ? 1 : 0; + tb.Append("<>__AnonType"); + tb.Append(hackTypeNumber); + tb.Append("<"); + tb.BuildList(",", type.GetMembers().OfType(), buildParam); + tb.Append(">"); + } + + /// + /// Constructs a display name string for this type symbol. + /// + /// The trap builder used to store the result. + public static void BuildDisplayName(this ITypeSymbol type, Context cx, ITrapBuilder tb) + { + using (cx.StackGuard) + { + switch (type.TypeKind) + { + case TypeKind.Array: + var array = (IArrayTypeSymbol)type; + var elementType = array.ElementType; + if (elementType.MetadataName.IndexOf("`") >= 0) + { + tb.Append(elementType.Name); + return; + } + elementType.BuildDisplayName(cx, tb); + array.BuildArraySuffix(tb); + return; + case TypeKind.Class: + case TypeKind.Interface: + case TypeKind.Struct: + case TypeKind.Enum: + case TypeKind.Delegate: + case TypeKind.Error: + var named = (INamedTypeSymbol)type; + named.BuildNamedTypeDisplayName(cx, tb); + return; + case TypeKind.Pointer: + var ptr = (IPointerTypeSymbol)type; + ptr.PointedAtType.BuildDisplayName(cx, tb); + tb.Append("*"); + return; + case TypeKind.TypeParameter: + tb.Append(type.Name); + return; + case TypeKind.Dynamic: + tb.Append("dynamic"); + return; + default: + throw new InternalError(type, "Unhandled type kind '{0}'", type.TypeKind); + } + } + } + + public static void BuildNamedTypeDisplayName(this INamedTypeSymbol namedType, Context cx, ITrapBuilder tb) + { + if (namedType.IsTupleType) + { + tb.Append("("); + tb.BuildList(",", namedType.TupleElements.Select(f => f.Type), + (t, tb0) => t.BuildDisplayName(cx, tb0) + ); + + tb.Append(")"); + return; + } + + if (namedType.IsAnonymousType) + { + namedType.BuildAnonymousName(cx, tb, (cx0, tb0, sub) => sub.BuildDisplayName(cx0, tb0), false); + } + + tb.Append(namedType.Name); + if (namedType.IsGenericType && namedType.TypeKind != TypeKind.Error && namedType.TypeArguments.Any()) + { + tb.Append("<"); + tb.BuildList(",", namedType.TypeArguments, (p, tb0) => + { + if (IsReallyBound(namedType)) + p.BuildDisplayName(cx, tb0); + }); + tb.Append(">"); + } + } + + public static bool IsReallyUnbound(this INamedTypeSymbol type) => + Equals(type.ConstructedFrom, type) || type.IsUnboundGenericType; + + public static bool IsReallyBound(this INamedTypeSymbol type) => !IsReallyUnbound(type); + + /// + /// Holds if this type is of the form int? or + /// System.Nullable. + /// + public static bool IsBoundNullable(this ITypeSymbol type) => + type.SpecialType == SpecialType.None && type.OriginalDefinition.IsUnboundNullable(); + + /// + /// Holds if this type is System.Nullable. + /// + public static bool IsUnboundNullable(this ITypeSymbol type) => + type.SpecialType == SpecialType.System_Nullable_T; + + /// + /// Gets the parameters of a method or property. + /// + /// The list of parameters, or an empty list. + public static IEnumerable GetParameters(this ISymbol parameterizable) + { + if (parameterizable is IMethodSymbol) + return ((IMethodSymbol)parameterizable).Parameters; + + if (parameterizable is IPropertySymbol) + return ((IPropertySymbol)parameterizable).Parameters; + + return Enumerable.Empty(); + } + + /// + /// Holds if this symbol is defined in a source code file. + /// + public static bool FromSource(this ISymbol symbol) => symbol.Locations.Any(l => l.IsInSource); + + /// + /// Holds if this symbol is a source declaration. + /// + public static bool IsSourceDeclaration(this ISymbol symbol) => Equals(symbol, symbol.OriginalDefinition); + + /// + /// Holds if this method is a source declaration. + /// + public static bool IsSourceDeclaration(this IMethodSymbol method) => + IsSourceDeclaration((ISymbol)method) && Equals(method, method.ConstructedFrom) && method.ReducedFrom == null; + + /// + /// Holds if this parameter is a source declaration. + /// + public static bool IsSourceDeclaration(this IParameterSymbol parameter) + { + var method = parameter.ContainingSymbol as IMethodSymbol; + if (method != null) + return method.IsSourceDeclaration(); + var property = parameter.ContainingSymbol as IPropertySymbol; + if (property != null && property.IsIndexer) + return property.IsSourceDeclaration(); + return true; + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Tuples.cs b/csharp/extractor/Semmle.Extraction.CSharp/Tuples.cs new file mode 100644 index 00000000000..2d118256a08 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Tuples.cs @@ -0,0 +1,215 @@ +using Semmle.Extraction.CommentProcessing; +using Semmle.Extraction.CSharp.Entities; +using Semmle.Extraction.Entities; +using Semmle.Extraction.Kinds; +using Semmle.Util; + +namespace Semmle.Extraction.CSharp +{ + /// + /// Methods for creating DB tuples. + /// + /// + /// + /// Notice how the parameters to the tuples are well typed. + /// In an idea world, each tuple would be its own type, as a typed entity. However + /// that seems to be overkill. + /// + internal static class Tuples + { + internal static Tuple accessor_location(Accessor accessorKey, Location location) => new Tuple("accessor_location", accessorKey, location); + + internal static Tuple accessors(Accessor accessorKey, int kind, string name, Property propKey, Accessor unboundAccessor) => new Tuple("accessors", accessorKey, kind, name, propKey, unboundAccessor); + + internal static Tuple array_element_type(ArrayType array, int dimension, int rank, Type elementType) => new Tuple("array_element_type", array, dimension, rank, elementType); + + internal static Tuple attributes(Attribute attribute, Type attributeType, IEntity entity) => new Tuple("attributes", attribute, attributeType, entity); + + internal static Tuple attribute_location(Attribute attribute, Location location) => new Tuple("attribute_location", attribute, location); + + internal static Tuple catch_type(Entities.Statements.Catch @catch, Type type, bool explicityCaught) => new Tuple("catch_type", @catch, type, explicityCaught ? 1 : 2); + + internal static Tuple commentblock(CommentBlock k) => new Tuple("commentblock", k); + + internal static Tuple commentblock_binding(CommentBlock commentBlock, Label entity, Binding binding) => new Tuple("commentblock_binding", commentBlock, entity, binding); + + internal static Tuple commentblock_child(CommentBlock commentBlock, CommentLine commentLine, int child) => new Tuple("commentblock_child", commentBlock, commentLine, child); + + internal static Tuple commentblock_location(CommentBlock k, Location l) => new Tuple("commentblock_location", k, l); + + internal static Tuple commentline(CommentLine commentLine, CommentType type, string text, string rawtext) => new Tuple("commentline", commentLine, type, text, rawtext); + + internal static Tuple commentline_location(CommentLine commentLine, Location location) => new Tuple("commentline_location", commentLine, location); + + internal static Tuple compiler_generated(IEntity entity) => new Tuple("compiler_generated", entity); + + internal static Tuple conditional_access(Expression access) => new Tuple("conditional_access", access); + + internal static Tuple constant_value(IEntity field, string value) => new Tuple("constant_value", field, value); + + internal static Tuple constructed_generic(IEntity constructedTypeOrMethod, IEntity unboundTypeOrMethod) => new Tuple("constructed_generic", constructedTypeOrMethod, unboundTypeOrMethod); + + internal static Tuple constructor_location(Constructor constructor, Location location) => new Tuple("constructor_location", constructor, location); + + internal static Tuple constructors(Constructor key, string name, Type definingType, Constructor originalDefinition) => new Tuple("constructors", key, name, definingType, originalDefinition); + + internal static Tuple delegate_return_type(Type delegateKey, Type returnType) => new Tuple("delegate_return_type", delegateKey, returnType); + + internal static Tuple destructor_location(Destructor destructor, Location location) => new Tuple("destructor_location", destructor, location); + + internal static Tuple destructors(Destructor destructor, string name, Type containingType, Destructor original) => new Tuple("destructors", destructor, name, containingType, original); + + internal static Tuple dynamic_member_name(Expression e, string name) => new Tuple("dynamic_member_name", e, name); + + internal static Tuple enum_underlying_type(Type @enum, Type type) => new Tuple("enum_underlying_type", @enum, type); + + internal static Tuple event_accessor_location(EventAccessor accessor, Location location) => new Tuple("event_accessor_location", accessor, location); + + internal static Tuple event_accessors(EventAccessor accessorKey, int type, string name, Event eventKey, EventAccessor unboundAccessor) => new Tuple("event_accessors", accessorKey, type, name, eventKey, unboundAccessor); + + internal static Tuple event_location(Event eventKey, Location locationKey) => new Tuple("event_location", eventKey, locationKey); + + internal static Tuple events(Event eventKey, string name, Type declaringType, Type memberType, Event originalDefinition) => new Tuple("events", eventKey, name, declaringType, memberType, originalDefinition); + + internal static Tuple explicitly_implements(IEntity member, Type @interface) => new Tuple("explicitly_implements", member, @interface); + + internal static Tuple explicitly_sized_array_creation(Expression array) => new Tuple("explicitly_sized_array_creation", array); + + internal static Tuple expr_compiler_generated(Expression expr) => new Tuple("expr_compiler_generated", expr); + + internal static Tuple expr_location(Expression exprKey, Location location) => new Tuple("expr_location", exprKey, location); + + internal static Tuple expr_access(Expression expr, IEntity access) => new Tuple("expr_access", expr, access); + + internal static Tuple expr_argument(Expression expr, int mode) => new Tuple("expr_argument", expr, mode); + + internal static Tuple expr_argument_name(Expression expr, string name) => new Tuple("expr_argument_name", expr, name); + + internal static Tuple expr_call(Expression expr, Method target) => new Tuple("expr_call", expr, target); + + internal static Tuple expr_parent(Expression exprKey, int child, IExpressionParentEntity parent) => new Tuple("expr_parent", exprKey, child, parent); + + internal static Tuple expr_parent_top_level(Expression exprKey, int child, IExpressionParentEntity parent) => new Tuple("expr_parent_top_level", exprKey, child, parent); + + internal static Tuple expr_value(Expression exprKey, string value) => new Tuple("expr_value", exprKey, value); + + internal static Tuple expressions(Expression expr, ExprKind kind, Type exprType) => new Tuple("expressions", expr, kind, exprType); + + internal static Tuple exprorstmt_name(IEntity expr, string name) => new Tuple("exprorstmt_name", expr, name); + + internal static Tuple extend(Type type, Type super) => new Tuple("extend", type, super); + + internal static Tuple field_location(Field field, Location location) => new Tuple("field_location", field, location); + + internal static Tuple fields(Field field, int @const, string name, Type declaringType, Type fieldType, Field unboundKey) => new Tuple("fields", field, @const, name, declaringType, fieldType, unboundKey); + + + internal static Tuple general_type_parameter_constraints(TypeParameterConstraints constraints, int hasKind) => new Tuple("general_type_parameter_constraints", constraints, hasKind); + + internal static Tuple has_modifiers(IEntity entity, Modifier modifier) => new Tuple("has_modifiers", entity, modifier); + + internal static Tuple implement(Type type, Type @interface) => new Tuple("implement", type, @interface); + + internal static Tuple implicitly_typed_array_creation(Expression array) => new Tuple("implicitly_typed_array_creation", array); + + internal static Tuple indexer_location(Indexer indexer, Location location) => new Tuple("indexer_location", indexer, location); + + internal static Tuple indexers(Indexer propKey, string name, Type declaringType, Type memberType, Indexer unboundProperty) => new Tuple("indexers", propKey, name, declaringType, memberType, unboundProperty); + + internal static Tuple is_constructed(IEntity typeOrMethod) => new Tuple("is_constructed", typeOrMethod); + + internal static Tuple is_generic(IEntity typeOrMethod) => new Tuple("is_generic", typeOrMethod); + + internal static Tuple jump_step(IEntity origin, IEntity src, Statement dest) => new Tuple("jump_step", origin, src, dest); + + internal static Tuple local_function_stmts(Entities.Statements.LocalFunction fnStmt, LocalFunction fn) => new Tuple("local_function_stmts", fnStmt, fn); + + internal static Tuple local_functions(LocalFunction fn, string name, Type returnType, LocalFunction unboundFn) => new Tuple("local_functions", fn, name, returnType, unboundFn); + + internal static Tuple localvar_location(LocalVariable var, Location location) => new Tuple("localvar_location", var, location); + + internal static Tuple localvars(LocalVariable key, int @const, string name, int @var, Type type, Expression expr) => new Tuple("localvars", key, @const, name, @var, type, expr); + + internal static Tuple method_location(Method method, Location location) => new Tuple("method_location", method, location); + + internal static Tuple methods(Method method, string name, Type declType, Type retType, Method originalDefinition) => new Tuple("methods", method, name, declType, retType, originalDefinition); + + internal static Tuple mutator_invocation_mode(Expression expr, int mode) => new Tuple("mutator_invocation_mode", expr, mode); + + internal static Tuple namespace_declaration_location(NamespaceDeclaration decl, Location location) => new Tuple("namespace_declaration_location", decl, location); + + internal static Tuple namespace_declarations(NamespaceDeclaration decl, Namespace ns) => new Tuple("namespace_declarations", decl, ns); + + internal static Tuple namespaces(Namespace ns, string name) => new Tuple("namespaces", ns, name); + + internal static Tuple nested_types(Type typeKey, Type declaringTypeKey, Type unboundTypeKey) => new Tuple("nested_types", typeKey, declaringTypeKey, unboundTypeKey); + + internal static Tuple nullable_underlying_type(Type nullableType, Type underlyingType) => new Tuple("nullable_underlying_type", nullableType, underlyingType); + + internal static Tuple numlines(IEntity label, LineCounts lineCounts) => new Tuple("numlines", label, lineCounts.Total, lineCounts.Code, lineCounts.Comment); + + internal static Tuple operator_location(UserOperator @operator, Location location) => new Tuple("operator_location", @operator, location); + + internal static Tuple operators(UserOperator method, string methodName, string symbol, Type classKey, Type returnType, UserOperator originalDefinition) => new Tuple("operators", method, methodName, symbol, classKey, returnType, originalDefinition); + + internal static Tuple overrides(Method overriding, Method overridden) => new Tuple("overrides", overriding, overridden); + + internal static Tuple param_location(Parameter param, Location location) => new Tuple("param_location", param, location); + + internal static Tuple @params(Parameter param, string name, Type type, int child, Parameter.Kind mode, IEntity method, Parameter originalDefinition) => new Tuple("params", param, name, type, child, mode, method, originalDefinition); + + internal static Tuple parent_namespace(IEntity type, Namespace parent) => new Tuple("parent_namespace", type, parent); + + internal static Tuple parent_namespace_declaration(IEntity item, NamespaceDeclaration parent) => new Tuple("parent_namespace_declaration", item, parent); + + internal static Tuple pointer_referent_type(PointerType pointerType, Type referentType) => new Tuple("pointer_referent_type", pointerType, referentType); + + internal static Tuple property_location(Property property, Location location) => new Tuple("property_location", property, location); + + internal static Tuple properties(Property propKey, string name, Type declaringType, Type memberType, Property unboundProperty) => new Tuple("properties", propKey, name, declaringType, memberType, unboundProperty); + + internal static Tuple ref_returns(IEntity method) => new Tuple("ref_returns", method); + + internal static Tuple ref_readonly_returns(IEntity method) => new Tuple("ref_readonly_returns", method); + + internal static Tuple statements(Statement stmt, StmtKind kind) => new Tuple("statements", stmt, kind); + + internal static Tuple specific_type_parameter_constraints(TypeParameterConstraints constraints, Type baseType) => new Tuple("specific_type_parameter_constraints", constraints, baseType); + + internal static Tuple successors(IEntity from, IEntity to) => new Tuple("successors", from, to); + + internal static Tuple stmt_location(Statement stmt, Location location) => new Tuple("stmt_location", stmt, location); + + internal static Tuple stmt_parent(Statement stmt, int child, IStatementParentEntity parent) => new Tuple("stmt_parent", stmt, child, parent); + + internal static Tuple stmt_parent_top_level(Statement stmt, int child, IStatementParentEntity parent) => new Tuple("stmt_parent_top_level", stmt, child, parent); + + internal static Tuple tuple_element(TupleType type, int index, Field field) => new Tuple("tuple_element", type, index, field); + + internal static Tuple tuple_underlying_type(TupleType type, NamedType underlying) => new Tuple("tuple_underlying_type", type, underlying); + + internal static Tuple type_mention(TypeMention ta, Type type, IEntity parent) => new Tuple("type_mention", ta, type, parent); + + internal static Tuple type_mention_location(TypeMention ta, Location loc) => new Tuple("type_mention_location", ta, loc); + + internal static Tuple type_arguments(Type arg, int n, IEntity typeOrMethod) => new Tuple("type_arguments", arg, n, typeOrMethod); + + internal static Tuple type_location(Type type, Location location) => new Tuple("type_location", type, location); + + internal static Tuple type_parameter_constraints(TypeParameterConstraints constraints, TypeParameter typeParam) => new Tuple("type_parameter_constraints", constraints, typeParam); + + internal static Tuple type_parameters(TypeParameter param, int child, IEntity typeOrMethod) => new Tuple("type_parameters", param, child, typeOrMethod, param.Variance); + + internal static Tuple typeref_type(NamedTypeRef typeref, Type type) => new Tuple("typeref_type", typeref, type); + + internal static Tuple typerefs(NamedTypeRef type, string name) => new Tuple("typerefs", type, name); + + internal static Tuple types(Type type, TypeKind kind, params string[] name) => new Tuple("types", type, kind, name); + + internal static Tuple using_namespace_directives(UsingDirective @using, Namespace ns) => new Tuple("using_namespace_directives", @using, ns); + + internal static Tuple using_directive_location(UsingDirective @using, Location location) => new Tuple("using_directive_location", @using, location); + + internal static Tuple using_static_directives(UsingDirective @using, Type type) => new Tuple("using_static_directives", @using, type); + } +} diff --git a/csharp/extractor/Semmle.Extraction.Tests/Layout.cs b/csharp/extractor/Semmle.Extraction.Tests/Layout.cs new file mode 100644 index 00000000000..631a7b89a07 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.Tests/Layout.cs @@ -0,0 +1,200 @@ +using System.IO; +using Xunit; +using Semmle.Util.Logging; +using System.Runtime.InteropServices; + +namespace Semmle.Extraction.Tests +{ + public class Layout + { + readonly ILogger Logger = new LoggerMock(); + + [Fact] + public void TestDefaultLayout() + { + var layout = new Semmle.Extraction.Layout(null, null, null); + var project = layout.LookupProjectOrNull("foo.cs"); + + // All files are mapped when there's no layout file. + Assert.True(layout.FileInLayout("foo.cs")); + + // Test trap filename + var tmpDir = Path.GetTempPath(); + Directory.SetCurrentDirectory(tmpDir); + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + // `Directory.SetCurrentDirectory()` doesn't seem to work on macOS, + // so disable this test on macOS, for now + Assert.NotEqual(Directory.GetCurrentDirectory(), tmpDir); + return; + } + var f1 = project.GetTrapPath(Logger, "foo.cs"); + var g1 = TrapWriter.NestPaths(Logger, tmpDir, "foo.cs.trap.gz", TrapWriter.InnerPathComputation.ABSOLUTE); + Assert.Equal(f1, g1); + + // Test trap file generation + var trapwriterFilename = project.GetTrapPath(Logger, "foo.cs"); + using (var trapwriter = project.CreateTrapWriter(Logger, "foo.cs", false)) + { + trapwriter.Emit("1=*"); + Assert.False(File.Exists(trapwriterFilename)); + } + Assert.True(File.Exists(trapwriterFilename)); + File.Delete(trapwriterFilename); + } + + [Fact] + public void TestLayoutFile() + { + File.WriteAllLines("layout.txt", new string[] + { + "# Section", + "TRAP_FOLDER=" + Path.GetFullPath("snapshot\\trap"), + "ODASA_DB=snapshot\\db-csharp", + "SOURCE_ARCHIVE=" + Path.GetFullPath("snapshot\\archive"), + "ODASA_BUILD_ERROR_DIR=snapshot\build-errors", + "-foo.cs", + "bar.cs", + "-excluded", + "excluded/foo.cs", + "included" + }); + + var layout = new Semmle.Extraction.Layout(null, null, "layout.txt"); + + // Test general pattern matching + Assert.True(layout.FileInLayout("bar.cs")); + Assert.False(layout.FileInLayout("foo.cs")); + Assert.False(layout.FileInLayout("goo.cs")); + Assert.False(layout.FileInLayout("excluded/bar.cs")); + Assert.True(layout.FileInLayout("excluded/foo.cs")); + Assert.True(layout.FileInLayout("included/foo.cs")); + + // Test the trap file + var project = layout.LookupProjectOrNull("bar.cs"); + var trapwriterFilename = project.GetTrapPath(Logger, "bar.cs"); + Assert.Equal(TrapWriter.NestPaths(Logger, Path.GetFullPath("snapshot\\trap"), "bar.cs.trap.gz", TrapWriter.InnerPathComputation.ABSOLUTE), + trapwriterFilename); + + // Test the source archive + var trapWriter = project.CreateTrapWriter(Logger, "bar.cs", false); + trapWriter.Archive("layout.txt", System.Text.Encoding.ASCII); + var writtenFile = TrapWriter.NestPaths(Logger, Path.GetFullPath("snapshot\\archive"), "layout.txt", TrapWriter.InnerPathComputation.ABSOLUTE); + Assert.True(File.Exists(writtenFile)); + File.Delete("layout.txt"); + } + + [Fact] + public void TestTrapOverridesLayout() + { + // When you specify both a trap file and a layout, use the trap file. + var layout = new Semmle.Extraction.Layout(Path.GetFullPath("snapshot\\trap"), null, "something.txt"); + Assert.True(layout.FileInLayout("bar.cs")); + var f1 = layout.LookupProjectOrNull("foo.cs").GetTrapPath(Logger, "foo.cs"); + var g1 = TrapWriter.NestPaths(Logger, Path.GetFullPath("snapshot\\trap"), "foo.cs.trap.gz", TrapWriter.InnerPathComputation.ABSOLUTE); + Assert.Equal(f1, g1); + } + + [Fact] + public void TestMultipleSections() + { + File.WriteAllLines("layout.txt", new string[] + { + "# Section 1", + "TRAP_FOLDER=" + Path.GetFullPath("snapshot\\trap1"), + "ODASA_DB=snapshot\\db-csharp", + "SOURCE_ARCHIVE=" + Path.GetFullPath("snapshot\\archive1"), + "ODASA_BUILD_ERROR_DIR=snapshot\build-errors", + "foo.cs", + "# Section 2", + "TRAP_FOLDER=" + Path.GetFullPath("snapshot\\trap2"), + "ODASA_DB=snapshot\\db-csharp", + "SOURCE_ARCHIVE=" + Path.GetFullPath("snapshot\\archive2"), + "ODASA_BUILD_ERROR_DIR=snapshot\build-errors", + "bar.cs", + }); + + var layout = new Semmle.Extraction.Layout(null, null, "layout.txt"); + + // Use Section 2 + Assert.True(layout.FileInLayout("bar.cs")); + var f1 = layout.LookupProjectOrNull("bar.cs").GetTrapPath(Logger, "bar.cs"); + var g1 = TrapWriter.NestPaths(Logger, Path.GetFullPath("snapshot\\trap2"), "bar.cs.trap.gz", TrapWriter.InnerPathComputation.ABSOLUTE); + Assert.Equal(f1, g1); + + // Use Section 1 + Assert.True(layout.FileInLayout("foo.cs")); + var f2 = layout.LookupProjectOrNull("foo.cs").GetTrapPath(Logger, "foo.cs"); + var g2 = TrapWriter.NestPaths(Logger, Path.GetFullPath("snapshot\\trap1"), "foo.cs.trap.gz", TrapWriter.InnerPathComputation.ABSOLUTE); + Assert.Equal(f2, g2); + + // boo.dll is not in the layout, so use layout from first section. + Assert.False(layout.FileInLayout("boo.dll")); + var f3 = layout.LookupProjectOrDefault("boo.dll").GetTrapPath(Logger, "boo.dll"); + var g3 = TrapWriter.NestPaths(Logger, Path.GetFullPath("snapshot\\trap1"), "boo.dll.trap.gz", TrapWriter.InnerPathComputation.ABSOLUTE); + Assert.Equal(f3, g3); + + // boo.cs is not in the layout, so return null + Assert.False(layout.FileInLayout("boo.cs")); + Assert.Null(layout.LookupProjectOrNull("boo.cs")); + } + + [Fact] + public void MissingLayout() + { + Assert.Throws(() => + new Semmle.Extraction.Layout(null, null, "nosuchfile.txt")); + } + + [Fact] + public void EmptyLayout() + { + File.Create("layout.txt").Close(); + Assert.Throws(() => + new Semmle.Extraction.Layout(null, null, "layout.txt")); + } + + [Fact] + public void InvalidLayout() + { + File.WriteAllLines("layout.txt", new string[] + { + "# Section 1" + }); + + Assert.Throws(() => + new Semmle.Extraction.Layout(null, null, "layout.txt")); + } + + class LoggerMock : ILogger + { + public void Dispose() { } + + public void Log(Severity s, string text) { } + + public void Log(Severity s, string text, params object[] args) { } + } + } + + static class TrapWriterTestExtensions + { + public static void Emit(this TrapWriter tw, string s) + { + tw.Emit(new StringTrapEmitter(s)); + } + + class StringTrapEmitter : ITrapEmitter + { + string Content; + public StringTrapEmitter(string content) + { + Content = content; + } + + public void EmitToTrapBuilder(ITrapBuilder tb) + { + tb.Append(Content); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.Tests/Options.cs b/csharp/extractor/Semmle.Extraction.Tests/Options.cs new file mode 100644 index 00000000000..50ea47ae979 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.Tests/Options.cs @@ -0,0 +1,188 @@ +using Xunit; +using Semmle.Util.Logging; +using System; + +namespace Semmle.Extraction.Tests +{ + public class OptionsTests + { + CSharp.Options options; + CSharp.Standalone.Options standaloneOptions; + + public OptionsTests() + { + Environment.SetEnvironmentVariable("SEMMLE_EXTRACTOR_OPTIONS", ""); + Environment.SetEnvironmentVariable("LGTM_INDEX_EXTRACTOR", ""); + } + + [Fact] + public void DefaultOptions() + { + options = CSharp.Options.CreateWithEnvironment(new string[] { }); + Assert.True(options.Cache); + Assert.False(options.CIL); + Assert.Null(options.Framework); + Assert.Null(options.CompilerName); + Assert.Empty(options.CompilerArguments); + Assert.True(options.Threads >= 1); + Assert.Equal(Verbosity.Info, options.Verbosity); + Assert.False(options.Console); + Assert.False(options.ClrTracer); + Assert.False(options.PDB); + Assert.False(options.Fast); + } + + [Fact] + public void Threads() + { + options = CSharp.Options.CreateWithEnvironment(new string[] { "--threads", "3" }); + Assert.Equal(3, options.Threads); + } + + [Fact] + public void Cache() + { + options = CSharp.Options.CreateWithEnvironment(new string[] { "--nocache" }); + Assert.False(options.Cache); + } + + [Fact] + public void CIL() + { + options = CSharp.Options.CreateWithEnvironment(new string[] { "--cil" }); + Assert.True(options.CIL); + options = CSharp.Options.CreateWithEnvironment(new string[] { "--cil", "--nocil" }); + Assert.False(options.CIL); + } + + [Fact] + public void CompilerArguments() + { + options = CSharp.Options.CreateWithEnvironment(new string[] { "x", "y", "z" }); + Assert.Equal("x", options.CompilerArguments[0]); + Assert.Equal("y", options.CompilerArguments[1]); + Assert.Equal("z", options.CompilerArguments[2]); + } + + [Fact] + public void VerbosityTests() + { + options = CSharp.Options.CreateWithEnvironment(new string[] { "--verbose" }); + Assert.Equal(Verbosity.Debug, options.Verbosity); + + options = CSharp.Options.CreateWithEnvironment(new string[] { "--verbosity", "0" }); + Assert.Equal(Verbosity.Off, options.Verbosity); + + options = CSharp.Options.CreateWithEnvironment(new string[] { "--verbosity", "1" }); + Assert.Equal(Verbosity.Error, options.Verbosity); + + options = CSharp.Options.CreateWithEnvironment(new string[] { "--verbosity", "2" }); + Assert.Equal(Verbosity.Warning, options.Verbosity); + + options = CSharp.Options.CreateWithEnvironment(new string[] { "--verbosity", "3" }); + Assert.Equal(Verbosity.Info, options.Verbosity); + + options = CSharp.Options.CreateWithEnvironment(new string[] { "--verbosity", "4" }); + Assert.Equal(Verbosity.Debug, options.Verbosity); + + options = CSharp.Options.CreateWithEnvironment(new string[] { "--verbosity", "5" }); + Assert.Equal(Verbosity.Trace, options.Verbosity); + + Assert.Throws(() => CSharp.Options.CreateWithEnvironment(new string[] { "--verbosity", "X" })); + } + + [Fact] + public void Console() + { + options = CSharp.Options.CreateWithEnvironment(new string[] { "--console" }); + Assert.True(options.Console); + } + + [Fact] + public void PDB() + { + options = CSharp.Options.CreateWithEnvironment(new string[] { "--pdb" }); + Assert.True(options.PDB); + } + + [Fact] + public void Compiler() + { + options = CSharp.Options.CreateWithEnvironment(new string[] { "--compiler", "foo" }); + Assert.Equal("foo", options.CompilerName); + } + + [Fact] + public void Framework() + { + options = CSharp.Options.CreateWithEnvironment(new string[] { "--framework", "foo" }); + Assert.Equal("foo", options.Framework); + } + + [Fact] + public void EnvironmentVariables() + { + Environment.SetEnvironmentVariable("SEMMLE_EXTRACTOR_OPTIONS", "--cil c"); + options = CSharp.Options.CreateWithEnvironment(new string[] { "a", "b" }); + Assert.True(options.CIL); + Assert.Equal("a", options.CompilerArguments[0]); + Assert.Equal("b", options.CompilerArguments[1]); + Assert.Equal("c", options.CompilerArguments[2]); + + Environment.SetEnvironmentVariable("SEMMLE_EXTRACTOR_OPTIONS", ""); + Environment.SetEnvironmentVariable("LGTM_INDEX_EXTRACTOR", "--nocil"); + options = CSharp.Options.CreateWithEnvironment(new string[] { "--cil" }); + Assert.False(options.CIL); + } + + [Fact] + public void StandaloneDefaults() + { + standaloneOptions = CSharp.Standalone.Options.Create(new string[] { }); + Assert.Equal(0, standaloneOptions.DllDirs.Count); + Assert.True(standaloneOptions.UseNuGet); + Assert.True(standaloneOptions.UseMscorlib); + Assert.False(standaloneOptions.SkipExtraction); + Assert.Null(standaloneOptions.SolutionFile); + Assert.True(standaloneOptions.ScanNetFrameworkDlls); + Assert.False(standaloneOptions.Errors); + } + + [Fact] + public void StandaloneOptions() + { + standaloneOptions = CSharp.Standalone.Options.Create(new string[] { "--references:foo", "--silent", "--skip-nuget", "--skip-dotnet", "--exclude", "bar", "--nostdlib" }); + Assert.Equal("foo", standaloneOptions.DllDirs[0]); + Assert.Equal("bar", standaloneOptions.Excludes[0]); + Assert.Equal(Verbosity.Off, standaloneOptions.Verbosity); + Assert.False(standaloneOptions.UseNuGet); + Assert.False(standaloneOptions.UseMscorlib); + Assert.False(standaloneOptions.ScanNetFrameworkDlls); + Assert.False(standaloneOptions.Errors); + Assert.False(standaloneOptions.Help); + } + + [Fact] + public void InvalidOptions() + { + standaloneOptions = CSharp.Standalone.Options.Create(new string[] { "--references:foo", "--silent", "--no-such-option" }); + Assert.True(standaloneOptions.Errors); + } + + [Fact] + public void ShowingHelp() + { + standaloneOptions = CSharp.Standalone.Options.Create(new string[] { "--help" }); + Assert.False(standaloneOptions.Errors); + Assert.True(standaloneOptions.Help); + } + + [Fact] + public void Fast() + { + Environment.SetEnvironmentVariable("LGTM_INDEX_EXTRACTOR", "--fast"); + options = CSharp.Options.CreateWithEnvironment(new string[] {}); + Assert.True(options.Fast); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.Tests/Properties/AssemblyInfo.cs b/csharp/extractor/Semmle.Extraction.Tests/Properties/AssemblyInfo.cs new file mode 100644 index 00000000000..f87cf947b4d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.Tests/Properties/AssemblyInfo.cs @@ -0,0 +1,35 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Semmle.Extraction.Tests")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Semmle.Extraction.Tests")] +[assembly: AssemblyCopyright("Copyright © 2016")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("23237396-31ef-41f8-b466-ee96ddd7b7bc")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/csharp/extractor/Semmle.Extraction.Tests/Semmle.Extraction.Tests.csproj b/csharp/extractor/Semmle.Extraction.Tests/Semmle.Extraction.Tests.csproj new file mode 100644 index 00000000000..24914080bcf --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.Tests/Semmle.Extraction.Tests.csproj @@ -0,0 +1,21 @@ + + + + Exe + netcoreapp2.0 + false + + + + + + + + + + + + + + + diff --git a/csharp/extractor/Semmle.Extraction.Tests/TrapWriter.cs b/csharp/extractor/Semmle.Extraction.Tests/TrapWriter.cs new file mode 100644 index 00000000000..fd7f77f427b --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.Tests/TrapWriter.cs @@ -0,0 +1,67 @@ +using Xunit; +using Semmle.Util.Logging; +using Semmle.Util; +using System.Runtime.InteropServices; +using System.IO; + +namespace Semmle.Extraction.Tests +{ + public class TrapWriterTests + { + [Fact] + public void NestedPaths() + { + string tempDir = System.IO.Path.GetTempPath(); + string root1, root2, root3; + + if(Win32.IsWindows()) + { + root1 = "E:"; + root2 = "e:"; + root3 = @"\"; + } + else + { + root1 = "/E_"; + root2 = "/e_"; + root3 = "/"; + } + + string formattedTempDir = tempDir.Replace('/', '\\').Replace(':', '_').Trim('\\'); + + var logger = new LoggerMock(); + System.IO.Directory.SetCurrentDirectory(tempDir); + + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + // `Directory.SetCurrentDirectory()` doesn't seem to work on macOS, + // so disable this test on macOS, for now + Assert.NotEqual(Directory.GetCurrentDirectory(), tempDir); + return; + } + + Assert.Equal($@"C:\Temp\source_archive\{formattedTempDir}\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", "def.cs", TrapWriter.InnerPathComputation.ABSOLUTE).Replace('/','\\')); + + Assert.Equal(@"C:\Temp\source_archive\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", "def.cs", TrapWriter.InnerPathComputation.RELATIVE).Replace('/', '\\')); + + Assert.Equal(@"C:\Temp\source_archive\E_\source\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", $@"{root1}\source\def.cs", TrapWriter.InnerPathComputation.ABSOLUTE).Replace('/', '\\')); + + Assert.Equal(@"C:\Temp\source_archive\e_\source\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", $@"{root2}\source\def.cs", TrapWriter.InnerPathComputation.RELATIVE).Replace('/', '\\')); + + Assert.Equal(@"C:\Temp\source_archive\source\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", $@"{root3}source\def.cs", TrapWriter.InnerPathComputation.ABSOLUTE).Replace('/', '\\')); + + Assert.Equal(@"C:\Temp\source_archive\source\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", $@"{root3}source\def.cs", TrapWriter.InnerPathComputation.RELATIVE).Replace('/', '\\')); + + Assert.Equal(@"C:\Temp\source_archive\diskstation\share\source\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", $@"{root3}{root3}diskstation\share\source\def.cs", TrapWriter.InnerPathComputation.ABSOLUTE).Replace('/', '\\')); + } + + class LoggerMock : ILogger + { + public void Dispose() { } + + public void Log(Severity s, string text) { } + + public void Log(Severity s, string text, params object[] args) { } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction/CommentProcessing.cs b/csharp/extractor/Semmle.Extraction/CommentProcessing.cs new file mode 100644 index 00000000000..443cc679122 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction/CommentProcessing.cs @@ -0,0 +1,482 @@ +using System.Collections.Generic; +using System.Linq; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Text; +using Semmle.Util; + +namespace Semmle.Extraction.CommentProcessing +{ + // The lexical type of the comment. + public enum CommentType + { + Singleline, // Comment starting // ... + XmlDoc, // Comment starting /// ... + Multiline, // Comment starting /* ..., even if the comment only spans one line. + MultilineContinuation // The second and subsequent lines of comment in a multiline comment. + }; + + // Relationship between a comment and a program element. + public enum Binding + { + Parent, // The parent element of a comment + Best, // The most likely element associated with a comment + Before, // The element before the comment + After // The element after the comment + }; + + /// + /// A single line of text in a comment. + /// + public interface ICommentLine + { + Location Location { get; } + CommentType Type { get; } + + // Trimmed text of the comment. + string Text { get; } + + // Complete text of the comment including leading/trailing whitespace and comment markers. + string RawText { get; } + } + + /// + /// A block of comment lines combined into one unit. + /// + public interface ICommentBlock + { + Location Location { get; } + IList CommentLines { get; } + } + + /// + /// Output for generated comment associations. + /// + /// The label of the element + /// The duplication guard key of the element, if any + /// The comment block associated with the element + /// The relationship between the commentblock and the element + public delegate void CommentBinding(Label elementLabel, Key duplicationGuardKey, ICommentBlock commentBlock, Binding binding); + + /// + /// Used by the populator to generate binding information between comments and program elements. + /// + public interface ICommentGenerator + { + /// + /// Registers the location of a program element to associate comments with. + /// Can be called in any order. + /// + /// Label of the element. + /// The duplication guard key of the element, if any. + /// Location of the element. + void RegisterElementLocation(Label elementLabel, Key duplicationGuardKey, Location location); + + void AddComment(ICommentLine comment); + + /// + /// Generate all binding information. + /// + /// Receiver of the binding information. + void GenerateBindings(CommentBinding cb); + } + + static class LocationExtension + { + public static int StartLine(this Location loc) => loc.GetLineSpan().Span.Start.Line; + + public static int StartColumn(this Location loc) => loc.GetLineSpan().Span.Start.Character; + + + public static int EndLine(this Location loc) => loc.GetLineSpan().Span.End.Line; + + /// + /// Whether one Location outer completely contains another Location inner. + /// + /// The outer location. + /// The inner location + /// Whether inner is completely container in outer. + public static bool Contains(this Location outer, Location inner) + { + bool sameFile = outer.SourceTree == inner.SourceTree; + bool startsBefore = outer.SourceSpan.Start <= inner.SourceSpan.Start; + bool endsAfter = outer.SourceSpan.End >= inner.SourceSpan.End; + return sameFile && startsBefore && endsAfter; + } + + /// + /// Whether one Location ends before another starts. + /// + /// The Location coming before + /// The Location coming after + /// Whether 'before' comes before 'after'. + public static bool Before(this Location before, Location after) + { + bool sameFile = before.SourceTree == after.SourceTree; + bool endsBefore = before.SourceSpan.End <= after.SourceSpan.Start; + return sameFile && endsBefore; + } + } + + /// + /// Implements the comment processor. + /// Registers locations of comments and program elements, + /// then generates binding information. + /// + class CommentProcessor : ICommentGenerator + { + public void AddComment(ICommentLine comment) + { + comments[comment.Location] = comment; + } + + // Comments sorted by location. + readonly SortedDictionary comments = new SortedDictionary(new LocationComparer()); + + // Program elements sorted by location. + readonly SortedDictionary elements = new SortedDictionary(new LocationComparer()); + + readonly Dictionary duplicationGuardKeys = new Dictionary(); + + Key GetDuplicationGuardKey(Label label) + { + Key duplicationGuardKey; + if (duplicationGuardKeys.TryGetValue(label, out duplicationGuardKey)) + return duplicationGuardKey; + return null; + } + + class LocationComparer : IComparer + { + public int Compare(Location l1, Location l2) => CommentProcessor.Compare(l1, l2); + } + + /// + /// Comparer for two locations, allowing them to be inserted into a sorted list. + /// + /// First location + /// Second location + /// <0 if l1 before l2, >0 if l1 after l2, else 0. + static int Compare(Location l1, Location l2) + { + int diff = l1.SourceTree == l2.SourceTree ? 0 : l1.SourceTree.FilePath.CompareTo(l2.SourceTree.FilePath); + if (diff != 0) return diff; + diff = l1.SourceSpan.Start - l2.SourceSpan.Start; + if (diff != 0) return diff; + return l1.SourceSpan.End - l2.SourceSpan.End; + } + + /// + /// Called by the populator when there is a program element which can have comments. + /// + /// The label of the element in the trap file. + /// The duplication guard key of the element, if any. + /// The location of the element. + public void RegisterElementLocation(Label elementLabel, Key duplicationGuardKey, Location loc) + { + if (loc != null && loc.IsInSource) + elements[loc] = elementLabel; + if (duplicationGuardKey != null) + duplicationGuardKeys[elementLabel] = duplicationGuardKey; + } + + // Ensure that commentBlock and element refer to the same file + // which can happen when processing multiple files. + void EnsureSameFile(ICommentBlock commentBlock, ref KeyValuePair? element) + { + if (element != null && element.Value.Key.SourceTree != commentBlock.Location.SourceTree) + element = null; + } + + /// + /// Generate the bindings between a comment and program elements. + /// Called once for each commentBlock. + /// + /// + /// The comment block. + /// The element before the comment block. + /// The element after the comment block. + /// The parent element of the comment block. + /// Output binding information. + void GenerateBindings( + ICommentBlock commentBlock, + KeyValuePair? previousElement, + KeyValuePair? nextElement, + KeyValuePair? parentElement, + CommentBinding cb + ) + { + EnsureSameFile(commentBlock, ref previousElement); + EnsureSameFile(commentBlock, ref nextElement); + EnsureSameFile(commentBlock, ref parentElement); + + if (previousElement != null) + { + var key = previousElement.Value.Value; + cb(key, GetDuplicationGuardKey(key), commentBlock, Binding.Before); + } + + if (nextElement != null) + { + var key = nextElement.Value.Value; + cb(key, GetDuplicationGuardKey(key), commentBlock, Binding.After); + } + + if (parentElement != null) + { + var key = parentElement.Value.Value; + cb(key, GetDuplicationGuardKey(key), commentBlock, Binding.Parent); + } + + // Heuristic to decide which is the "best" element associated with the comment. + KeyValuePair? bestElement; + + if (previousElement != null && previousElement.Value.Key.EndLine() == commentBlock.Location.StartLine()) + { + // 1. If the comment is on the same line as the previous element, use that + bestElement = previousElement; + } + else if (nextElement != null && nextElement.Value.Key.StartLine() == commentBlock.Location.EndLine()) + { + // 2. If the comment is on the same line as the next element, use that + bestElement = nextElement; + } + else if (nextElement != null && previousElement != null && + previousElement.Value.Key.EndLine() + 1 == commentBlock.Location.StartLine() && + commentBlock.Location.EndLine() + 1 == nextElement.Value.Key.StartLine()) + { + // 3. If comment is equally between two elements, use the parentElement + // because it's ambiguous whether the comment refers to the next or previous element + bestElement = parentElement; + } + else if (nextElement != null && nextElement.Value.Key.StartLine() == commentBlock.Location.EndLine() + 1) + { + // 4. If there is no gap after the comment, use "nextElement" + bestElement = nextElement; + } + else if (previousElement != null && previousElement.Value.Key.EndLine() + 1 == commentBlock.Location.StartLine()) + { + // 5. If there is no gap before the comment, use previousElement + bestElement = previousElement; + } + else + { + // 6. Otherwise, bind the comment to the parent block. + bestElement = parentElement; + + /* if parentElement==null, then there is no best element. The comment is effectively orphaned. + * + * This can be caused by comments that are not in a type declaration. + * Due to restrictions in the dbscheme, the comment cannot be associated with the "file" + * which is not an element, and the "using" declarations are not emitted by the extractor. + */ + } + + if (bestElement != null) + { + var label = bestElement.Value.Value; + cb(label, GetDuplicationGuardKey(label), commentBlock, Binding.Best); + } + } + + // Stores element nesting information in a stack. + // Top of stack = most nested element, based on Location. + class ElementStack + { + // Invariant: the top of the stack must be contained by items below it. + readonly Stack> elementStack = new Stack>(); + + /// + /// Add a new element to the stack. + /// + /// The stack is maintained. + /// The new element to push. + public void Push(KeyValuePair value) + { + // Maintain the invariant by popping existing elements + while (elementStack.Count > 0 && !elementStack.Peek().Key.Contains(value.Key)) + elementStack.Pop(); + + elementStack.Push(value); + } + + /// + /// Locate the parent of a comment with location l. + /// + /// The location of the comment. + /// An element completely containing l, or null if none found. + public KeyValuePair? FindParent(Location l) => + elementStack.Where(v => v.Key.Contains(l)).FirstOrNull(); + + /// + /// Finds the element on the stack immediately preceding the comment at l. + /// + /// The location of the comment. + /// The element before l, or null. + public KeyValuePair? FindBefore(Location l) + { + return elementStack. + Where(v => v.Key.SourceSpan.End < l.SourceSpan.Start). + LastOrNull(); + } + + /// + /// Finds the element after the comment. + /// + /// The location of the comment. + /// The next element. + /// The next element. + public KeyValuePair? FindAfter(Location comment, KeyValuePair? next) + { + var p = FindParent(comment); + return next.HasValue && p.HasValue && p.Value.Key.Before(next.Value.Key) ? null : next; + } + } + + // Generate binding information for one CommentBlock. + void GenerateBindings( + ICommentBlock block, + ElementStack elementStack, + KeyValuePair? nextElement, + CommentBinding cb + ) + { + if (block.CommentLines.Count > 0) + { + GenerateBindings( + block, + elementStack.FindBefore(block.Location), + elementStack.FindAfter(block.Location, nextElement), + elementStack.FindParent(block.Location), + cb); + } + } + + /// + /// Process comments up until nextElement. + /// Group comments into blocks, and associate blocks with elements. + /// + /// Enumerator for all comments in the program. + /// The next element in the list. + /// A stack of nested program elements. + /// Where to send the results. + /// true if there are more comments to process, false otherwise. + bool GenerateBindings( + IEnumerator> commentEnumerator, + KeyValuePair? nextElement, + ElementStack elementStack, + CommentBinding cb + ) + { + CommentBlock block = new CommentBlock(); + + // Iterate comments until the commentEnumerator has gone past nextElement + while (nextElement == null || Compare(commentEnumerator.Current.Value.Location, nextElement.Value.Key) < 0) + { + if (!block.CombinesWith(commentEnumerator.Current.Value)) + { + // Start of a new block, so generate the bindings for the old block first. + GenerateBindings(block, elementStack, nextElement, cb); + block = new CommentBlock(); + } + + block.AddCommentLine(commentEnumerator.Current.Value); + + // Get the next comment. + if (!commentEnumerator.MoveNext()) + { + // If there are no more comments, generate the remaining bindings and return false. + GenerateBindings(block, elementStack, nextElement, cb); + return false; + } + } + + GenerateBindings(block, elementStack, nextElement, cb); + return true; + } + + /// + /// Merge comments into blocks and associate comment blocks with program elements. + /// + /// Callback for the binding information + public void GenerateBindings(CommentBinding cb) + { + /* Algorithm: + * Do a merge of elements and comments, which are both sorted in location order. + * + * Iterate through all elements, and iterate all comment lines between adjacent pairs of elements. + * Maintain a stack of elements, such that the top of the stack must be fully nested in the + * element below it. This enables comments to be associated with the "parent" element, as well as + * elements before, after and "best" element match for a comment. + * + * This is an O(n) algorithm because the list of elements and comments are traversed once. + * (Note that comment processing is O(n.log n) overall due to dictionary of elements and comments.) + */ + + ElementStack elementStack = new ElementStack(); + + using (IEnumerator> elementEnumerator = elements.GetEnumerator()) + using (IEnumerator> commentEnumerator = comments.GetEnumerator()) + { + if (!commentEnumerator.MoveNext()) + { + // There are no comments to process. + return; + } + + while (elementEnumerator.MoveNext()) + { + if (!GenerateBindings(commentEnumerator, elementEnumerator.Current, elementStack, cb)) + { + // No more comments to process. + return; + } + + elementStack.Push(elementEnumerator.Current); + } + + // Generate remaining comments at end of file + GenerateBindings(commentEnumerator, null, elementStack, cb); + } + } + } + + class CommentBlock : ICommentBlock + { + public IList CommentLines { get; } = new List(); + + public Location Location { get; private set; } + + /// + /// Determine whether commentlines should be merged. + /// + /// A comment line to be appended to this comment block. + /// Whether the new line should be appended to this block. + public bool CombinesWith(ICommentLine newLine) + { + if (CommentLines.Count == 0) return true; + + bool sameFile = Location.SourceTree == newLine.Location.SourceTree; + bool sameRow = Location.EndLine() == newLine.Location.StartLine(); + bool sameColumn = Location.EndLine() + 1 == newLine.Location.StartLine(); + bool nextRow = Location.StartColumn() == newLine.Location.StartColumn(); + bool adjacent = sameFile && (sameRow || (sameColumn && nextRow)); + + return + newLine.Type == CommentType.MultilineContinuation || + adjacent; + } + + /// + /// Adds a comment line to the this comment block. + /// + /// The line to add. + public void AddCommentLine(ICommentLine line) + { + Location = CommentLines.Count == 0 ? + line.Location : + Location.Create(line.Location.SourceTree, new TextSpan(Location.SourceSpan.Start, line.Location.SourceSpan.End - Location.SourceSpan.Start)); + CommentLines.Add(line); + } + } +} + diff --git a/csharp/extractor/Semmle.Extraction/Context.cs b/csharp/extractor/Semmle.Extraction/Context.cs new file mode 100644 index 00000000000..b66370e896b --- /dev/null +++ b/csharp/extractor/Semmle.Extraction/Context.cs @@ -0,0 +1,496 @@ +using Microsoft.CodeAnalysis; +using System.Linq; +using Semmle.Extraction.CommentProcessing; +using System.Collections.Generic; +using System; +using Semmle.Util.Logging; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction +{ + /// + /// State which needs needs to be available throughout the extraction process. + /// There is one Context object per trap output file. + /// + public class Context + { + /// + /// Interface to various extraction functions, e.g. logger, trap writer. + /// + public readonly IExtractor Extractor; + + /// + /// The program database provided by Roslyn. + /// There's one per syntax tree, which makes things awkward. + /// + public SemanticModel Model(SyntaxNode node) + { + if (cachedModel == null || node.SyntaxTree != cachedModel.SyntaxTree) + { + cachedModel = Compilation.GetSemanticModel(node.SyntaxTree); + } + + return cachedModel; + } + + SemanticModel cachedModel; + + /// + /// Access to the trap file. + /// + public readonly TrapWriter TrapWriter; + + int NewId() => TrapWriter.IdCounter++; + + /// + /// Gets the cached label for the given entity, or creates a new + /// (cached) label if it hasn't already been created. The label + /// is set on the supplied object. + /// + /// true iff the label already existed. + public bool GetOrAddCachedLabel(ICachedEntity entity) + { + var id = GetId(entity); + if (id == null) + throw new InternalError("Attempt to create a null entity for {0}", entity.GetType()); + + Label existingLabel; + if (labelCache.TryGetValue(id, out existingLabel)) + { + entity.Label = existingLabel; + return true; + } + + entity.Label = new Label(NewId()); + DefineLabel(entity.Label, id); + labelCache[id] = entity.Label; + return false; + } + + /// + /// Should the given entity be extracted? + /// A second call to this method will always return false, + /// on the assumption that it would have been extracted on the first call. + /// + /// This is used to track the extraction of generics, which cannot be extracted + /// in a top-down manner. + /// + /// The entity to extract. + /// True only on the first call for a particular entity. + public bool ExtractGenerics(ICachedEntity entity) + { + if (extractedGenerics.Contains(entity.Label)) + { + return false; + } + else + { + extractedGenerics.Add(entity.Label); + return true; + } + } + + /// + /// Gets the ID belonging to cached entity . + /// + /// The ID itself is also cached, but unlike the label cache (which is used + /// to prevent reextraction/infinite loops), this is a pure performance + /// optimization. Moreover, the label cache is injective, which the ID cache + /// need not be. + /// + IId GetId(ICachedEntity entity) + { + IId id; + if (!idCache.TryGetValue(entity, out id)) + { + id = entity.Id; + idCache[entity] = id; + } + return id; + } + + /// + /// Creates a fresh label with ID "*", and set it on the + /// supplied object. + /// + public void AddFreshLabel(IEntity entity) + { + var label = new Label(NewId()); + TrapWriter.Emit(new DefineFreshLabelEmitter(label)); + entity.Label = label; + } + + readonly Dictionary labelCache = new Dictionary(); + readonly HashSet