hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-22 07:15:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into rdmarsh2/swift/for-in

Browse Source
This commit is contained in:
Robert Marsh
2023-09-26 15:12:46 +00:00
parent 811c33b016 7c230d61a8
commit ae1072e09f
689 changed files with 94532 additions and 78051 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.github/labeler.yml vendored
View File

@@ -45,11 +45,7 @@ documentation:
# Since these are all shared files that need to be synced, just pick _one_ copy of each.
"DataFlow Library":
- "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll"
- "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll"
- "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
- "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll"
- "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll"
- "shared/dataflow/**/*"
"ATM":
- javascript/ql/experimental/adaptivethreatmodeling/**/*

4
config/identical-files.json
View File

@@ -28,8 +28,6 @@
"python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll",
"ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl1.qll",
"ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll",
"ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForHttpClientLibraries.qll",
"ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForPathname.qll",
"swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl1.qll"
],
"TaintTracking Legacy Configuration Java/C++/C#/Go/Python/Ruby/Swift": [
@@ -552,4 +550,4 @@
"python/ql/test/experimental/dataflow/model-summaries/InlineTaintTest.ext.yml",
"python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.ext.yml"
]
}
}

2213
cpp/downgrades/dbe9c8eb5fc6f54b7ae08c7317d0795b24961564/old.dbscheme Normal file
View File

File diff suppressed because it is too large Load Diff

2212
cpp/downgrades/dbe9c8eb5fc6f54b7ae08c7317d0795b24961564/semmlecode.cpp.dbscheme Normal file
View File

File diff suppressed because it is too large Load Diff

2
cpp/downgrades/dbe9c8eb5fc6f54b7ae08c7317d0795b24961564/upgrade.properties Normal file
View File

@@ -0,0 +1,2 @@
description: Make __is_trivial a builtin operation
compatibility: full

33
cpp/ql/lib/semmle/code/cpp/controlflow/Dereferenced.qll
View File

@@ -26,17 +26,18 @@ predicate callDereferences(FunctionCall fc, int i) {
}
/**
* Holds if evaluation of `op` dereferences `e`.
* Holds if evaluation of `op` dereferences `e` directly.
*
* This predicate does not recurse through function calls or arithmetic operations. To find
* such cases, use `dereferencedByOperation`.
*/
predicate dereferencedByOperation(Expr op, Expr e) {
predicate directDereferencedByOperation(Expr op, Expr e) {
exists(PointerDereferenceExpr deref |
deref.getAChild() = e and
deref = op and
not deref.getParent*() instanceof SizeofOperator
)
or
exists(CrementOperation crement | dereferencedByOperation(e, op) and crement.getOperand() = e)
or
exists(ArrayExpr ae |
(
not ae.getParent() instanceof AddressOfExpr and
@@ -50,6 +51,24 @@ predicate dereferencedByOperation(Expr op, Expr e) {
)
)
or
// ptr->Field
e = op.(FieldAccess).getQualifier() and isClassPointerType(e.getType())
or
// ptr->method()
e = op.(Call).getQualifier() and isClassPointerType(e.getType())
}
/**
* Holds if evaluation of `op` dereferences `e`.
*
* This includes the set of operations identified via `directDereferencedByOperation`, as well
* as calls to function that are known to dereference an argument.
*/
predicate dereferencedByOperation(Expr op, Expr e) {
directDereferencedByOperation(op, e)
or
exists(CrementOperation crement | dereferencedByOperation(e, op) and crement.getOperand() = e)
or
exists(AddressOfExpr addof, ArrayExpr ae |
dereferencedByOperation(addof, op) and
addof.getOperand() = ae and
@@ -74,12 +93,6 @@ predicate dereferencedByOperation(Expr op, Expr e) {
e = fc.getArgument(i) and
op = fc
)
or
// ptr->Field
e = op.(FieldAccess).getQualifier() and isClassPointerType(e.getType())
or
// ptr->method()
e = op.(Call).getQualifier() and isClassPointerType(e.getType())
}
private predicate isClassPointerType(Type t) {

2
cpp/ql/lib/semmle/code/cpp/controlflow/IRGuards.qll
View File

@@ -240,7 +240,7 @@ private class GuardConditionFromIR extends GuardCondition {
*/
private predicate controlsBlock(BasicBlock controlled, boolean testIsTrue) {
exists(IRBlock irb |
forex(IRGuardCondition inst | inst = ir | inst.controls(irb, testIsTrue)) and
ir.controls(irb, testIsTrue) and
irb.getAnInstruction().getAst().(ControlFlowNode).getBasicBlock() = controlled and
not isUnreachedBlock(irb)
)

10
cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowDispatch.qll
View File

@@ -79,13 +79,3 @@ class ArgumentPosition extends int {
/** Holds if arguments at position `apos` match parameters at position `ppos`. */
pragma[inline]
predicate parameterMatch(ParameterPosition ppos, ArgumentPosition apos) { ppos = apos }
/**
* Holds if flow from `call`'s argument `arg` to parameter `p` is permissible.
*
* This is a temporary hook to support technical debt in the Go language; do not use.
*/
pragma[inline]
predicate golangSpecificParamArgFilter(DataFlowCall call, ParameterNode p, ArgumentNode arg) {
any()
}

11
cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowPrivate.qll
View File

@@ -208,6 +208,8 @@ predicate expectsContent(Node n, ContentSet c) { none() }
predicate typeStrongerThan(DataFlowType t1, DataFlowType t2) { none() }
predicate localMustFlowStep(Node node1, Node node2) { none() }
/** Gets the type of `n` used for type pruning. */
Type getNodeType(Node n) {
suppressUnusedNode(n) and
@@ -295,12 +297,3 @@ class ContentApprox = Unit;
/** Gets an approximated value for content `c`. */
pragma[inline]
ContentApprox getContentApprox(Content c) { any() }
/**
* Gets an additional term that is added to the `join` and `branch` computations to reflect
* an additional forward or backwards branching factor that is not taken into account
* when calculating the (virtual) dispatch cost.
*
* Argument `arg` is part of a path from a source to a sink, and `p` is the target parameter.
*/
int getAdditionalFlowIntoCallNodeTerm(ArgumentNode arg, ParameterNode p) { none() }

18
cpp/ql/lib/semmle/code/cpp/exprs/BuiltInOperations.qll
View File

@@ -1547,3 +1547,21 @@ class BuiltInBitCast extends BuiltInOperation, @builtinbitcast {
override string getAPrimaryQlClass() { result = "BuiltInBitCast" }
}
/**
* A C++ `__is_trivial` built-in operation (used by some implementations of the
* `<type_traits>` header).
*
* Returns `true` if a type is a trivial type.
* ```
* template<typename _Tp>
* struct is_trivial
* : public integral_constant<bool, __is_trivial(_Tp)>
* {};
* ```
*/
class BuiltInIsTrivial extends BuiltInOperation, @istrivialexpr {
override string toString() { result = "__is_trivial" }
override string getAPrimaryQlClass() { result = "BuiltInIsTrivial" }
}

10
cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowDispatch.qll
View File

@@ -271,13 +271,3 @@ DataFlowCallable viableImplInCallContext(DataFlowCall call, DataFlowCall ctx) {
/** Holds if arguments at position `apos` match parameters at position `ppos`. */
pragma[inline]
predicate parameterMatch(ParameterPosition ppos, ArgumentPosition apos) { ppos = apos }
/**
* Holds if flow from `call`'s argument `arg` to parameter `p` is permissible.
*
* This is a temporary hook to support technical debt in the Go language; do not use.
*/
pragma[inline]
predicate golangSpecificParamArgFilter(DataFlowCall call, ParameterNode p, ArgumentNode arg) {
any()
}

2
cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplSpecific.qll
View File

@@ -18,4 +18,6 @@ module CppDataFlow implements InputSig {
import Public
Node exprNode(DataFlowExpr e) { result = Public::exprNode(e) }
predicate getAdditionalFlowIntoCallNodeTerm = Private::getAdditionalFlowIntoCallNodeTerm/2;
}

2
cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll
View File

@@ -804,6 +804,8 @@ predicate expectsContent(Node n, ContentSet c) { none() }
predicate typeStrongerThan(DataFlowType t1, DataFlowType t2) { none() }
predicate localMustFlowStep(Node node1, Node node2) { none() }
/** Gets the type of `n` used for type pruning. */
DataFlowType getNodeType(Node n) {
suppressUnusedNode(n) and

27
cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll
View File

@@ -1066,15 +1066,28 @@ private module GetConvertedResultExpression {
private import semmle.code.cpp.ir.implementation.raw.internal.TranslatedExpr
private import semmle.code.cpp.ir.implementation.raw.internal.InstructionTag
private Operand getAnInitializeDynamicAllocationInstructionAddress() {
result = any(InitializeDynamicAllocationInstruction init).getAllocationAddressOperand()
}
/**
* Gets the expression that should be returned as the result expression from `instr`.
*
* Note that this predicate may return multiple results in cases where a conversion belond to a
* Note that this predicate may return multiple results in cases where a conversion belongs to a
* different AST element than its operand.
*/
Expr getConvertedResultExpression(Instruction instr, int n) {
// Only fully converted instructions has a result for `asConvertedExpr`
not conversionFlow(unique( | | getAUse(instr)), _, false, false) and
// Only fully converted instructions have a result for `asConvertedExpr`
not conversionFlow(unique(Operand op |
// The address operand of a `InitializeDynamicAllocationInstruction` is
// special: we need to handle it during dataflow (since it's
// effectively a store to an indirection), but it doesn't appear in
// source syntax, so dataflow node <-> expression conversion shouldn't
// care about it.
op = getAUse(instr) and not op = getAnInitializeDynamicAllocationInstructionAddress()
|
op
), _, false, false) and
result = getConvertedResultExpressionImpl(instr) and
n = 0
or
@@ -1341,6 +1354,9 @@ class ParameterNode extends Node {
* pointer-indirection parameters are at further negative positions.
*/
predicate isParameterOf(Function f, ParameterPosition pos) { none() } // overridden by subclasses
/** Gets the `Parameter` associated with this node, if it exists. */
Parameter getParameter() { none() } // overridden by subclasses
}
/** An explicit positional parameter, including `this`, but not `...`. */
@@ -1363,10 +1379,9 @@ private class ExplicitParameterNode extends ParameterNode, DirectParameterNode {
f.getParameter(pos.(DirectPosition).getIndex()) = instr.getParameter()
}
/** Gets the `Parameter` associated with this node. */
Parameter getParameter() { result = instr.getParameter() }
override string toStringImpl() { result = instr.getParameter().toString() }
override Parameter getParameter() { result = instr.getParameter() }
}
/** An implicit `this` parameter. */

2
cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternals.qll
View File

@@ -766,7 +766,7 @@ predicate fromPhiNode(SsaPhiNode nodeFrom, Node nodeTo) {
or
exists(PhiNode phiTo |
phi != phiTo and
lastRefRedefExt(phi, _, _, phiTo) and
lastRefRedefExt(phi, bb1, i1, phiTo) and
nodeTo.(SsaPhiNode).getPhiNode() = phiTo
)
)

25
cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedExpr.qll
View File

@@ -1906,8 +1906,10 @@ class TranslatedNonConstantAllocationSize extends TranslatedAllocationSize {
final override predicate hasInstruction(Opcode opcode, InstructionTag tag, CppType resultType) {
resultType = getTypeForPRValue(expr.getAllocator().getParameter(0).getType()) and
(
this.extentNeedsConversion() and
// Convert the extent to `size_t`, because the AST doesn't do this already.
tag = AllocationExtentConvertTag() and opcode instanceof Opcode::Convert
tag = AllocationExtentConvertTag() and
opcode instanceof Opcode::Convert
or
tag = AllocationElementSizeTag() and opcode instanceof Opcode::Constant
or
@@ -1918,6 +1920,7 @@ class TranslatedNonConstantAllocationSize extends TranslatedAllocationSize {
final override Instruction getInstructionSuccessor(InstructionTag tag, EdgeKind kind) {
kind instanceof GotoEdge and
(
this.extentNeedsConversion() and
tag = AllocationExtentConvertTag() and
result = this.getInstruction(AllocationElementSizeTag())
or
@@ -1933,7 +1936,9 @@ class TranslatedNonConstantAllocationSize extends TranslatedAllocationSize {
final override Instruction getChildSuccessor(TranslatedElement child) {
child = this.getExtent() and
result = this.getInstruction(AllocationExtentConvertTag())
if this.extentNeedsConversion()
then result = this.getInstruction(AllocationExtentConvertTag())
else result = this.getInstruction(AllocationElementSizeTag())
}
final override string getInstructionConstantValue(InstructionTag tag) {
@@ -1945,18 +1950,32 @@ class TranslatedNonConstantAllocationSize extends TranslatedAllocationSize {
tag = AllocationSizeTag() and
(
operandTag instanceof LeftOperandTag and
result = this.getInstruction(AllocationExtentConvertTag())
(
if this.extentNeedsConversion()
then result = this.getInstruction(AllocationExtentConvertTag())
else result = this.getExtent().getResult()
)
or
operandTag instanceof RightOperandTag and
result = this.getInstruction(AllocationElementSizeTag())
)
or
this.extentNeedsConversion() and
tag = AllocationExtentConvertTag() and
operandTag instanceof UnaryOperandTag and
result = this.getExtent().getResult()
}
TranslatedExpr getExtent() { result = getTranslatedExpr(expr.getExtent().getFullyConverted()) }
/**
* Holds if the result of `expr.getExtent()` does not have the same type as
* the allocator's size parameter.
*/
private predicate extentNeedsConversion() {
expr.getExtent().getFullyConverted().getUnspecifiedType() !=
expr.getAllocator().getParameter(0).getUnspecifiedType()
}
}
/**

49
cpp/ql/lib/semmle/code/cpp/rangeanalysis/new/internal/semantic/analysis/RangeAnalysisStage.qll
View File

@@ -917,25 +917,46 @@ module RangeStage<
bounded(cast.getOperand(), b, delta, upper, fromBackEdge, origdelta, reason)
}
pragma[nomagic]
private predicate initialBoundedUpper(SemExpr e) {
exists(D::Delta d |
initialBounded(e, _, d, false, _, _, _) and
D::toFloat(d) >= 0
)
}
private predicate noOverflow0(SemExpr e, boolean upper) {
exists(boolean lower | lower = upper.booleanNot() |
semExprDoesNotOverflow(lower, e)
or
upper = [true, false] and
not potentiallyOverflowingExpr(lower, e)
)
}
pragma[nomagic]
private predicate initialBoundedLower(SemExpr e) {
exists(D::Delta d |
initialBounded(e, _, d, true, _, _, _) and
D::toFloat(d) <= 0
)
}
pragma[nomagic]
private predicate noOverflow(SemExpr e, boolean upper) {
noOverflow0(e, upper)
or
upper = true and initialBoundedUpper(e)
or
upper = false and initialBoundedLower(e)
}
predicate bounded(
SemExpr e, SemBound b, D::Delta delta, boolean upper, boolean fromBackEdge, D::Delta origdelta,
SemReason reason
) {
initialBounded(e, b, delta, upper, fromBackEdge, origdelta, reason) and
(
semExprDoesNotOverflow(upper.booleanNot(), e)
or
not potentiallyOverflowingExpr(upper.booleanNot(), e)
or
exists(D::Delta otherDelta |
initialBounded(e, _, otherDelta, upper.booleanNot(), _, _, _) and
(
upper = true and D::toFloat(otherDelta) >= 0
or
upper = false and D::toFloat(otherDelta) <= 0
)
)
)
noOverflow(e, upper)
}
predicate potentiallyOverflowingExpr(boolean positively, SemExpr expr) {

1
cpp/ql/lib/semmlecode.cpp.dbscheme
View File

@@ -1755,6 +1755,7 @@ case @expr.kind of
| @istriviallydestructibleexpr
| @istriviallyassignableexpr
| @isnothrowassignableexpr
| @istrivialexpr
| @isstandardlayoutexpr
| @istriviallycopyableexpr
| @isliteraltypeexpr

2212
cpp/ql/lib/upgrades/d77c09d8bdc172c9201dec293de1e14c931d3f05/old.dbscheme Normal file
View File

File diff suppressed because it is too large Load Diff

2213
cpp/ql/lib/upgrades/d77c09d8bdc172c9201dec293de1e14c931d3f05/semmlecode.cpp.dbscheme Normal file
View File

File diff suppressed because it is too large Load Diff

2
cpp/ql/lib/upgrades/d77c09d8bdc172c9201dec293de1e14c931d3f05/upgrade.properties Normal file
View File

@@ -0,0 +1,2 @@
description: Make __is_trivial a builtin operation
compatibility: full

18
cpp/ql/src/Metrics/Internal/ASTConsistency.ql Normal file
View File

@@ -0,0 +1,18 @@
/**
* @name Count AST inconsistencies
* @description Counts the various AST inconsistencies that may occur.
* This query is for internal use only and may change without notice.
* @kind table
* @id cpp/count-ast-inconsistencies
*/
import cpp
predicate hasDuplicateFunctionEntryPointLocation(Function func) {
count(func.getEntryPoint().getLocation()) > 1
}
predicate hasDuplicateFunctionEntryPoint(Function func) { count(func.getEntryPoint()) > 1 }
select count(Function f | hasDuplicateFunctionEntryPoint(f) | f) as duplicateFunctionEntryPoint,
count(Function f | hasDuplicateFunctionEntryPointLocation(f) | f) as duplicateFunctionEntryPointLocation

7
cpp/ql/test/library-tests/builtins/type_traits/clang.cpp
View File

@@ -86,3 +86,10 @@ bool bok_is_void2 = __is_void(int);
bool bok_is_volatile1 = __is_volatile(volatile int);
bool bok_is_volatile2 = __is_volatile(int);
struct S2 {
S2() {}
};
bool bok_is_trivial1 = __is_trivial(int);
bool bok_is_trivial2 = __is_trivial(S2);

4
cpp/ql/test/library-tests/builtins/type_traits/expr.expected
View File

@@ -121,6 +121,10 @@
| clang.cpp:87:25:87:51 | volatile int | | <none> |
| clang.cpp:88:25:88:42 | __is_volatile | int | 0 |
| clang.cpp:88:25:88:42 | int | | <none> |
| clang.cpp:94:24:94:40 | __is_trivial | int | 1 |
| clang.cpp:94:24:94:40 | int | | <none> |
| clang.cpp:95:24:95:39 | S2 | | <none> |
| clang.cpp:95:24:95:39 | __is_trivial | S2 | 0 |
| file://:0:0:0:0 | 0 | | 0 |
| file://:0:0:0:0 | 1 | | 1 |
| file://:0:0:0:0 | 2 | | 2 |

19
cpp/ql/test/library-tests/dataflow/dataflow-tests/guard-condition-regression-test.cpp Normal file
View File

@@ -0,0 +1,19 @@
int source();
void gard_condition_sink(int);
void use(int);
/*
This test checks that we hit the node corresponding to the expression node that wraps `source`
in the condition `source >= 0`.
*/
void test_guard_condition(int source, bool b)
{
if (b) {
use(source);
}
if (source >= 0) {
use(source);
}
gard_condition_sink(source); // clean
}

2
cpp/ql/test/library-tests/dataflow/dataflow-tests/guard-condition-regression-test.expected Normal file
View File

@@ -0,0 +1,2 @@
testFailures
failures

40
cpp/ql/test/library-tests/dataflow/dataflow-tests/guard-condition-regression-test.ql Normal file
View File

@@ -0,0 +1,40 @@
import TestUtilities.InlineExpectationsTest
private import cpp
private import semmle.code.cpp.ir.dataflow.DataFlow
private import semmle.code.cpp.controlflow.IRGuards
module IRTestAllocationConfig implements DataFlow::ConfigSig {
predicate isSource(DataFlow::Node source) {
source.asParameter().getName().matches("source%") and
source.getLocation().getFile().getBaseName() = "guard-condition-regression-test.cpp"
}
predicate isSink(DataFlow::Node sink) {
exists(FunctionCall call, Expr e | e = call.getAnArgument() |
call.getTarget().getName() = "gard_condition_sink" and
sink.asExpr() = e
)
}
predicate isBarrier(DataFlow::Node node) {
exists(GuardCondition gc | node.asExpr() = gc.getAChild*())
}
}
private module Flow = DataFlow::Global<IRTestAllocationConfig>;
module GuardConditionRegressionTest implements TestSig {
string getARelevantTag() { result = "guard-condition-regression" }
predicate hasActualResult(Location location, string element, string tag, string value) {
exists(DataFlow::Node sink |
Flow::flowTo(sink) and
location = sink.getLocation() and
element = sink.toString() and
tag = "guard-condition-regression" and
value = ""
)
}
}
import MakeTest<GuardConditionRegressionTest>

2
cpp/ql/test/library-tests/dataflow/dataflow-tests/test.expected
View File

@@ -5,5 +5,5 @@ WARNING: Module DataFlow has been deprecated and may be removed in future (test.
WARNING: Module DataFlow has been deprecated and may be removed in future (test.ql:40,25-33)
WARNING: Module DataFlow has been deprecated and may be removed in future (test.ql:42,17-25)
WARNING: Module DataFlow has been deprecated and may be removed in future (test.ql:46,20-28)
failures
testFailures
failures

89
cpp/ql/test/library-tests/ir/ir/PrintAST.expected
View File

@@ -15846,6 +15846,95 @@ ir.cpp:
# 2095| getExpr(): [VariableAccess] x
# 2095| Type = [IntType] int
# 2095| ValueCategory = prvalue(load)
# 2098| [TopLevelFunction] void newArrayCorrectType(size_t)
# 2098| <params>:
# 2098| getParameter(0): [Parameter] n
# 2098| Type = [CTypedefType,Size_t] size_t
# 2098| getEntryPoint(): [BlockStmt] { ... }
# 2099| getStmt(0): [ExprStmt] ExprStmt
# 2099| getExpr(): [NewArrayExpr] new[]
# 2099| Type = [IntPointerType] int *
# 2099| ValueCategory = prvalue
# 2099| getExtent(): [VariableAccess] n
# 2099| Type = [CTypedefType,Size_t] size_t
# 2099| ValueCategory = prvalue(load)
# 2100| getStmt(1): [ExprStmt] ExprStmt
# 2100| getExpr(): [NewArrayExpr] new[]
# 2100| Type = [IntPointerType] int *
# 2100| ValueCategory = prvalue
# 2100| getAllocatorCall(): [FunctionCall] call to operator new[]
# 2100| Type = [VoidPointerType] void *
# 2100| ValueCategory = prvalue
# 2100| getArgument(0): [ErrorExpr] <error expr>
# 2100| Type = [LongType] unsigned long
# 2100| ValueCategory = prvalue
# 2100| getArgument(1): [Literal] 1.0
# 2100| Type = [FloatType] float
# 2100| Value = [Literal] 1.0
# 2100| ValueCategory = prvalue
# 2100| getExtent(): [VariableAccess] n
# 2100| Type = [CTypedefType,Size_t] size_t
# 2100| ValueCategory = prvalue(load)
# 2101| getStmt(2): [ExprStmt] ExprStmt
# 2101| getExpr(): [NewArrayExpr] new[]
# 2101| Type = [PointerType] String *
# 2101| ValueCategory = prvalue
# 2101| getInitializer(): [ArrayAggregateLiteral] {...}
# 2101| Type = [ArrayType] String[]
# 2101| ValueCategory = prvalue
# 2101| getAnElementExpr(0): [ConstructorCall] call to String
# 2101| Type = [VoidType] void
# 2101| ValueCategory = prvalue
# 2101| getExtent(): [VariableAccess] n
# 2101| Type = [CTypedefType,Size_t] size_t
# 2101| ValueCategory = prvalue(load)
# 2102| getStmt(3): [ExprStmt] ExprStmt
# 2102| getExpr(): [NewArrayExpr] new[]
# 2102| Type = [PointerType] Overaligned *
# 2102| ValueCategory = prvalue
# 2102| getExtent(): [VariableAccess] n
# 2102| Type = [CTypedefType,Size_t] size_t
# 2102| ValueCategory = prvalue(load)
# 2102| getAlignmentArgument(): [Literal] 128
# 2102| Type = [ScopedEnum] align_val_t
# 2102| Value = [Literal] 128
# 2102| ValueCategory = prvalue
# 2103| getStmt(4): [ExprStmt] ExprStmt
# 2103| getExpr(): [NewArrayExpr] new[]
# 2103| Type = [PointerType] DefaultCtorWithDefaultParam *
# 2103| ValueCategory = prvalue
# 2103| getInitializer(): [ArrayAggregateLiteral] {...}
# 2103| Type = [ArrayType] DefaultCtorWithDefaultParam[]
# 2103| ValueCategory = prvalue
# 2103| getAnElementExpr(0): [ConstructorCall] call to DefaultCtorWithDefaultParam
# 2103| Type = [VoidType] void
# 2103| ValueCategory = prvalue
# 2103| getExtent(): [VariableAccess] n
# 2103| Type = [CTypedefType,Size_t] size_t
# 2103| ValueCategory = prvalue(load)
# 2104| getStmt(5): [ExprStmt] ExprStmt
# 2104| getExpr(): [NewArrayExpr] new[]
# 2104| Type = [IntPointerType] int *
# 2104| ValueCategory = prvalue
# 2104| getInitializer(): [ArrayAggregateLiteral] {...}
# 2104| Type = [ArrayType] int[3]
# 2104| ValueCategory = prvalue
# 2104| getAnElementExpr(0): [Literal] 0
# 2104| Type = [IntType] int
# 2104| Value = [Literal] 0
# 2104| ValueCategory = prvalue
# 2104| getAnElementExpr(1): [Literal] 1
# 2104| Type = [IntType] int
# 2104| Value = [Literal] 1
# 2104| ValueCategory = prvalue
# 2104| getAnElementExpr(2): [Literal] 2
# 2104| Type = [IntType] int
# 2104| Value = [Literal] 2
# 2104| ValueCategory = prvalue
# 2104| getExtent(): [VariableAccess] n
# 2104| Type = [CTypedefType,Size_t] size_t
# 2104| ValueCategory = prvalue(load)
# 2105| getStmt(6): [ReturnStmt] return ...
perf-regression.cpp:
# 4| [CopyAssignmentOperator] Big& Big::operator=(Big const&)
# 4| <params>:

75
cpp/ql/test/library-tests/ir/ir/aliased_ir.expected
View File

@@ -12215,6 +12215,81 @@ ir.cpp:
# 2090| v2090_8(void) = AliasedUse : ~m2094_4
# 2090| v2090_9(void) = ExitFunction :
# 2098| void newArrayCorrectType(size_t)
# 2098| Block 0
# 2098| v2098_1(void) = EnterFunction :
# 2098| m2098_2(unknown) = AliasedDefinition :
# 2098| m2098_3(unknown) = InitializeNonLocal :
# 2098| m2098_4(unknown) = Chi : total:m2098_2, partial:m2098_3
# 2098| r2098_5(glval<unsigned long>) = VariableAddress[n] :
# 2098| m2098_6(unsigned long) = InitializeParameter[n] : &:r2098_5
# 2099| r2099_1(glval<unknown>) = FunctionAddress[operator new[]] :
# 2099| r2099_2(glval<unsigned long>) = VariableAddress[n] :
# 2099| r2099_3(unsigned long) = Load[n] : &:r2099_2, m2098_6
# 2099| r2099_4(unsigned long) = Constant[4] :
# 2099| r2099_5(unsigned long) = Mul : r2099_3, r2099_4
# 2099| r2099_6(void *) = Call[operator new[]] : func:r2099_1, 0:r2099_5
# 2099| m2099_7(unknown) = ^CallSideEffect : ~m2098_4
# 2099| m2099_8(unknown) = Chi : total:m2098_4, partial:m2099_7
# 2099| m2099_9(unknown) = ^InitializeDynamicAllocation : &:r2099_6
# 2099| r2099_10(int *) = Convert : r2099_6
# 2100| r2100_1(glval<unknown>) = FunctionAddress[operator new[]] :
# 2100| r2100_2(glval<unsigned long>) = VariableAddress[n] :
# 2100| r2100_3(unsigned long) = Load[n] : &:r2100_2, m2098_6
# 2100| r2100_4(unsigned long) = Constant[4] :
# 2100| r2100_5(unsigned long) = Mul : r2100_3, r2100_4
# 2100| r2100_6(float) = Constant[1.0] :
# 2100| r2100_7(void *) = Call[operator new[]] : func:r2100_1, 0:r2100_5, 1:r2100_6
# 2100| m2100_8(unknown) = ^CallSideEffect : ~m2099_8
# 2100| m2100_9(unknown) = Chi : total:m2099_8, partial:m2100_8
# 2100| m2100_10(unknown) = ^InitializeDynamicAllocation : &:r2100_7
# 2100| r2100_11(int *) = Convert : r2100_7
# 2101| r2101_1(glval<unknown>) = FunctionAddress[operator new[]] :
# 2101| r2101_2(glval<unsigned long>) = VariableAddress[n] :
# 2101| r2101_3(unsigned long) = Load[n] : &:r2101_2, m2098_6
# 2101| r2101_4(unsigned long) = Constant[8] :
# 2101| r2101_5(unsigned long) = Mul : r2101_3, r2101_4
# 2101| r2101_6(void *) = Call[operator new[]] : func:r2101_1, 0:r2101_5
# 2101| m2101_7(unknown) = ^CallSideEffect : ~m2100_9
# 2101| m2101_8(unknown) = Chi : total:m2100_9, partial:m2101_7
# 2101| m2101_9(unknown) = ^InitializeDynamicAllocation : &:r2101_6
# 2101| r2101_10(String *) = Convert : r2101_6
# 2102| r2102_1(glval<unknown>) = FunctionAddress[operator new[]] :
# 2102| r2102_2(glval<unsigned long>) = VariableAddress[n] :
# 2102| r2102_3(unsigned long) = Load[n] : &:r2102_2, m2098_6
# 2102| r2102_4(unsigned long) = Constant[256] :
# 2102| r2102_5(unsigned long) = Mul : r2102_3, r2102_4
# 2102| r2102_6(align_val_t) = Constant[128] :
# 2102| r2102_7(void *) = Call[operator new[]] : func:r2102_1, 0:r2102_5, 1:r2102_6
# 2102| m2102_8(unknown) = ^CallSideEffect : ~m2101_8
# 2102| m2102_9(unknown) = Chi : total:m2101_8, partial:m2102_8
# 2102| m2102_10(unknown) = ^InitializeDynamicAllocation : &:r2102_7
# 2102| r2102_11(Overaligned *) = Convert : r2102_7
# 2103| r2103_1(glval<unknown>) = FunctionAddress[operator new[]] :
# 2103| r2103_2(glval<unsigned long>) = VariableAddress[n] :
# 2103| r2103_3(unsigned long) = Load[n] : &:r2103_2, m2098_6
# 2103| r2103_4(unsigned long) = Constant[1] :
# 2103| r2103_5(unsigned long) = Mul : r2103_3, r2103_4
# 2103| r2103_6(void *) = Call[operator new[]] : func:r2103_1, 0:r2103_5
# 2103| m2103_7(unknown) = ^CallSideEffect : ~m2102_9
# 2103| m2103_8(unknown) = Chi : total:m2102_9, partial:m2103_7
# 2103| m2103_9(unknown) = ^InitializeDynamicAllocation : &:r2103_6
# 2103| r2103_10(DefaultCtorWithDefaultParam *) = Convert : r2103_6
# 2104| r2104_1(glval<unknown>) = FunctionAddress[operator new[]] :
# 2104| r2104_2(glval<unsigned long>) = VariableAddress[n] :
# 2104| r2104_3(unsigned long) = Load[n] : &:r2104_2, m2098_6
# 2104| r2104_4(unsigned long) = Constant[4] :
# 2104| r2104_5(unsigned long) = Mul : r2104_3, r2104_4
# 2104| r2104_6(void *) = Call[operator new[]] : func:r2104_1, 0:r2104_5
# 2104| m2104_7(unknown) = ^CallSideEffect : ~m2103_8
# 2104| m2104_8(unknown) = Chi : total:m2103_8, partial:m2104_7
# 2104| m2104_9(unknown) = ^InitializeDynamicAllocation : &:r2104_6
# 2104| r2104_10(int *) = Convert : r2104_6
# 2105| v2105_1(void) = NoOp :
# 2098| v2098_7(void) = ReturnVoid :
# 2098| v2098_8(void) = AliasedUse : ~m2104_8
# 2098| v2098_9(void) = ExitFunction :
perf-regression.cpp:
# 6| void Big::Big()
# 6| Block 0

9
cpp/ql/test/library-tests/ir/ir/ir.cpp
View File

@@ -2095,4 +2095,13 @@ int TransNonExit() {
return x;
}
void newArrayCorrectType(size_t n) {
new int[n]; // No constructor
new(1.0f) int[n]; // Placement new, no constructor
new String[n]; // Constructor
new Overaligned[n]; // Aligned new
new DefaultCtorWithDefaultParam[n];
new int[n] { 0, 1, 2 };
}
// semmle-extractor-options: -std=c++17 --clang

72
cpp/ql/test/library-tests/ir/ir/operand_locations.expected
View File

@@ -9896,6 +9896,78 @@
| ir.cpp:2095:12:2095:12 | Address | &:r2095_2 |
| ir.cpp:2095:12:2095:12 | Load | m2091_8 |
| ir.cpp:2095:12:2095:12 | StoreValue | r2095_3 |
| ir.cpp:2098:6:2098:24 | ChiPartial | partial:m2098_3 |
| ir.cpp:2098:6:2098:24 | ChiTotal | total:m2098_2 |
| ir.cpp:2098:6:2098:24 | SideEffect | ~m2104_8 |
| ir.cpp:2098:33:2098:33 | Address | &:r2098_5 |
| ir.cpp:2099:3:2099:12 | Address | &:r2099_6 |
| ir.cpp:2099:3:2099:12 | Arg(0) | 0:r2099_5 |
| ir.cpp:2099:3:2099:12 | CallTarget | func:r2099_1 |
| ir.cpp:2099:3:2099:12 | ChiPartial | partial:m2099_7 |
| ir.cpp:2099:3:2099:12 | ChiTotal | total:m2098_4 |
| ir.cpp:2099:3:2099:12 | Right | r2099_4 |
| ir.cpp:2099:3:2099:12 | SideEffect | ~m2098_4 |
| ir.cpp:2099:3:2099:12 | Unary | r2099_6 |
| ir.cpp:2099:11:2099:11 | Address | &:r2099_2 |
| ir.cpp:2099:11:2099:11 | Left | r2099_3 |
| ir.cpp:2099:11:2099:11 | Load | m2098_6 |
| ir.cpp:2100:3:2100:18 | Address | &:r2100_7 |
| ir.cpp:2100:3:2100:18 | Arg(0) | 0:r2100_5 |
| ir.cpp:2100:3:2100:18 | CallTarget | func:r2100_1 |
| ir.cpp:2100:3:2100:18 | ChiPartial | partial:m2100_8 |
| ir.cpp:2100:3:2100:18 | ChiTotal | total:m2099_8 |
| ir.cpp:2100:3:2100:18 | Right | r2100_4 |
| ir.cpp:2100:3:2100:18 | SideEffect | ~m2099_8 |
| ir.cpp:2100:3:2100:18 | Unary | r2100_7 |
| ir.cpp:2100:7:2100:10 | Arg(1) | 1:r2100_6 |
| ir.cpp:2100:17:2100:17 | Address | &:r2100_2 |
| ir.cpp:2100:17:2100:17 | Left | r2100_3 |
| ir.cpp:2100:17:2100:17 | Load | m2098_6 |
| ir.cpp:2101:3:2101:15 | Address | &:r2101_6 |
| ir.cpp:2101:3:2101:15 | Arg(0) | 0:r2101_5 |
| ir.cpp:2101:3:2101:15 | CallTarget | func:r2101_1 |
| ir.cpp:2101:3:2101:15 | ChiPartial | partial:m2101_7 |
| ir.cpp:2101:3:2101:15 | ChiTotal | total:m2100_9 |
| ir.cpp:2101:3:2101:15 | Right | r2101_4 |
| ir.cpp:2101:3:2101:15 | SideEffect | ~m2100_9 |
| ir.cpp:2101:3:2101:15 | Unary | r2101_6 |
| ir.cpp:2101:14:2101:14 | Address | &:r2101_2 |
| ir.cpp:2101:14:2101:14 | Left | r2101_3 |
| ir.cpp:2101:14:2101:14 | Load | m2098_6 |
| ir.cpp:2102:3:2102:20 | Address | &:r2102_7 |
| ir.cpp:2102:3:2102:20 | Arg(0) | 0:r2102_5 |
| ir.cpp:2102:3:2102:20 | CallTarget | func:r2102_1 |
| ir.cpp:2102:3:2102:20 | ChiPartial | partial:m2102_8 |
| ir.cpp:2102:3:2102:20 | ChiTotal | total:m2101_8 |
| ir.cpp:2102:3:2102:20 | Right | r2102_4 |
| ir.cpp:2102:3:2102:20 | SideEffect | ~m2101_8 |
| ir.cpp:2102:3:2102:20 | Unary | r2102_7 |
| ir.cpp:2102:19:2102:19 | Address | &:r2102_2 |
| ir.cpp:2102:19:2102:19 | Left | r2102_3 |
| ir.cpp:2102:19:2102:19 | Load | m2098_6 |
| ir.cpp:2102:21:2102:21 | Arg(1) | 1:r2102_6 |
| ir.cpp:2103:3:2103:36 | Address | &:r2103_6 |
| ir.cpp:2103:3:2103:36 | Arg(0) | 0:r2103_5 |
| ir.cpp:2103:3:2103:36 | CallTarget | func:r2103_1 |
| ir.cpp:2103:3:2103:36 | ChiPartial | partial:m2103_7 |
| ir.cpp:2103:3:2103:36 | ChiTotal | total:m2102_9 |
| ir.cpp:2103:3:2103:36 | Right | r2103_4 |
| ir.cpp:2103:3:2103:36 | SideEffect | ~m2102_9 |
| ir.cpp:2103:3:2103:36 | Unary | r2103_6 |
| ir.cpp:2103:35:2103:35 | Address | &:r2103_2 |
| ir.cpp:2103:35:2103:35 | Left | r2103_3 |
| ir.cpp:2103:35:2103:35 | Load | m2098_6 |
| ir.cpp:2104:3:2104:24 | Address | &:r2104_6 |
| ir.cpp:2104:3:2104:24 | Arg(0) | 0:r2104_5 |
| ir.cpp:2104:3:2104:24 | CallTarget | func:r2104_1 |
| ir.cpp:2104:3:2104:24 | ChiPartial | partial:m2104_7 |
| ir.cpp:2104:3:2104:24 | ChiTotal | total:m2103_8 |
| ir.cpp:2104:3:2104:24 | Right | r2104_4 |
| ir.cpp:2104:3:2104:24 | SideEffect | ~m2103_8 |
| ir.cpp:2104:3:2104:24 | Unary | r2104_6 |
| ir.cpp:2104:11:2104:11 | Address | &:r2104_2 |
| ir.cpp:2104:11:2104:11 | Left | r2104_3 |
| ir.cpp:2104:11:2104:11 | Load | m2098_6 |
| perf-regression.cpp:6:3:6:5 | Address | &:r6_5 |
| perf-regression.cpp:6:3:6:5 | Address | &:r6_5 |
| perf-regression.cpp:6:3:6:5 | Address | &:r6_7 |

68
cpp/ql/test/library-tests/ir/ir/raw_ir.expected
View File

@@ -11429,6 +11429,74 @@ ir.cpp:
# 2090| v2090_6(void) = AliasedUse : ~m?
# 2090| v2090_7(void) = ExitFunction :
# 2098| void newArrayCorrectType(size_t)
# 2098| Block 0
# 2098| v2098_1(void) = EnterFunction :
# 2098| mu2098_2(unknown) = AliasedDefinition :
# 2098| mu2098_3(unknown) = InitializeNonLocal :
# 2098| r2098_4(glval<unsigned long>) = VariableAddress[n] :
# 2098| mu2098_5(unsigned long) = InitializeParameter[n] : &:r2098_4
# 2099| r2099_1(glval<unknown>) = FunctionAddress[operator new[]] :
# 2099| r2099_2(glval<unsigned long>) = VariableAddress[n] :
# 2099| r2099_3(unsigned long) = Load[n] : &:r2099_2, ~m?
# 2099| r2099_4(unsigned long) = Constant[4] :
# 2099| r2099_5(unsigned long) = Mul : r2099_3, r2099_4
# 2099| r2099_6(void *) = Call[operator new[]] : func:r2099_1, 0:r2099_5
# 2099| mu2099_7(unknown) = ^CallSideEffect : ~m?
# 2099| mu2099_8(unknown) = ^InitializeDynamicAllocation : &:r2099_6
# 2099| r2099_9(int *) = Convert : r2099_6
# 2100| r2100_1(glval<unknown>) = FunctionAddress[operator new[]] :
# 2100| r2100_2(glval<unsigned long>) = VariableAddress[n] :
# 2100| r2100_3(unsigned long) = Load[n] : &:r2100_2, ~m?
# 2100| r2100_4(unsigned long) = Constant[4] :
# 2100| r2100_5(unsigned long) = Mul : r2100_3, r2100_4
# 2100| r2100_6(float) = Constant[1.0] :
# 2100| r2100_7(void *) = Call[operator new[]] : func:r2100_1, 0:r2100_5, 1:r2100_6
# 2100| mu2100_8(unknown) = ^CallSideEffect : ~m?
# 2100| mu2100_9(unknown) = ^InitializeDynamicAllocation : &:r2100_7
# 2100| r2100_10(int *) = Convert : r2100_7
# 2101| r2101_1(glval<unknown>) = FunctionAddress[operator new[]] :
# 2101| r2101_2(glval<unsigned long>) = VariableAddress[n] :
# 2101| r2101_3(unsigned long) = Load[n] : &:r2101_2, ~m?
# 2101| r2101_4(unsigned long) = Constant[8] :
# 2101| r2101_5(unsigned long) = Mul : r2101_3, r2101_4
# 2101| r2101_6(void *) = Call[operator new[]] : func:r2101_1, 0:r2101_5
# 2101| mu2101_7(unknown) = ^CallSideEffect : ~m?
# 2101| mu2101_8(unknown) = ^InitializeDynamicAllocation : &:r2101_6
# 2101| r2101_9(String *) = Convert : r2101_6
# 2102| r2102_1(glval<unknown>) = FunctionAddress[operator new[]] :
# 2102| r2102_2(glval<unsigned long>) = VariableAddress[n] :
# 2102| r2102_3(unsigned long) = Load[n] : &:r2102_2, ~m?
# 2102| r2102_4(unsigned long) = Constant[256] :
# 2102| r2102_5(unsigned long) = Mul : r2102_3, r2102_4
# 2102| r2102_6(align_val_t) = Constant[128] :
# 2102| r2102_7(void *) = Call[operator new[]] : func:r2102_1, 0:r2102_5, 1:r2102_6
# 2102| mu2102_8(unknown) = ^CallSideEffect : ~m?
# 2102| mu2102_9(unknown) = ^InitializeDynamicAllocation : &:r2102_7
# 2102| r2102_10(Overaligned *) = Convert : r2102_7
# 2103| r2103_1(glval<unknown>) = FunctionAddress[operator new[]] :
# 2103| r2103_2(glval<unsigned long>) = VariableAddress[n] :
# 2103| r2103_3(unsigned long) = Load[n] : &:r2103_2, ~m?
# 2103| r2103_4(unsigned long) = Constant[1] :
# 2103| r2103_5(unsigned long) = Mul : r2103_3, r2103_4
# 2103| r2103_6(void *) = Call[operator new[]] : func:r2103_1, 0:r2103_5
# 2103| mu2103_7(unknown) = ^CallSideEffect : ~m?
# 2103| mu2103_8(unknown) = ^InitializeDynamicAllocation : &:r2103_6
# 2103| r2103_9(DefaultCtorWithDefaultParam *) = Convert : r2103_6
# 2104| r2104_1(glval<unknown>) = FunctionAddress[operator new[]] :
# 2104| r2104_2(glval<unsigned long>) = VariableAddress[n] :
# 2104| r2104_3(unsigned long) = Load[n] : &:r2104_2, ~m?
# 2104| r2104_4(unsigned long) = Constant[4] :
# 2104| r2104_5(unsigned long) = Mul : r2104_3, r2104_4
# 2104| r2104_6(void *) = Call[operator new[]] : func:r2104_1, 0:r2104_5
# 2104| mu2104_7(unknown) = ^CallSideEffect : ~m?
# 2104| mu2104_8(unknown) = ^InitializeDynamicAllocation : &:r2104_6
# 2104| r2104_9(int *) = Convert : r2104_6
# 2105| v2105_1(void) = NoOp :
# 2098| v2098_6(void) = ReturnVoid :
# 2098| v2098_7(void) = AliasedUse : ~m?
# 2098| v2098_8(void) = ExitFunction :
perf-regression.cpp:
# 6| void Big::Big()
# 6| Block 0

14
cpp/ql/test/query-tests/Security/CWE/CWE-119/SAMATE/OverrunWriteProductFlow.expected
View File

@@ -47,25 +47,19 @@ edges
| test.cpp:207:17:207:19 | str indirection [string] | test.cpp:207:22:207:27 | string |
| test.cpp:214:24:214:24 | p | test.cpp:216:10:216:10 | p |
| test.cpp:220:27:220:54 | call to malloc | test.cpp:222:15:222:20 | buffer |
| test.cpp:220:43:220:48 | call to malloc | test.cpp:222:15:222:20 | buffer |
| test.cpp:222:15:222:20 | buffer | test.cpp:214:24:214:24 | p |
| test.cpp:228:27:228:54 | call to malloc | test.cpp:232:10:232:15 | buffer |
| test.cpp:228:43:228:48 | call to malloc | test.cpp:232:10:232:15 | buffer |
| test.cpp:235:40:235:45 | buffer | test.cpp:236:5:236:26 | ... = ... |
| test.cpp:236:5:236:26 | ... = ... | test.cpp:236:12:236:17 | p_str indirection [post update] [string] |
| test.cpp:241:20:241:38 | call to malloc | test.cpp:242:22:242:27 | buffer |
| test.cpp:241:27:241:32 | call to malloc | test.cpp:242:22:242:27 | buffer |
| test.cpp:242:16:242:19 | set_string output argument [string] | test.cpp:243:12:243:14 | str indirection [string] |
| test.cpp:242:22:242:27 | buffer | test.cpp:235:40:235:45 | buffer |
| test.cpp:242:22:242:27 | buffer | test.cpp:242:16:242:19 | set_string output argument [string] |
| test.cpp:243:12:243:14 | str indirection [string] | test.cpp:243:12:243:21 | string |
| test.cpp:249:14:249:33 | call to my_alloc | test.cpp:250:12:250:12 | p |
| test.cpp:256:9:256:25 | call to malloc | test.cpp:257:12:257:12 | p |
| test.cpp:256:17:256:22 | call to malloc | test.cpp:257:12:257:12 | p |
| test.cpp:262:15:262:30 | call to malloc | test.cpp:266:12:266:12 | p |
| test.cpp:262:22:262:27 | call to malloc | test.cpp:266:12:266:12 | p |
| test.cpp:264:13:264:30 | call to malloc | test.cpp:266:12:266:12 | p |
| test.cpp:264:20:264:25 | call to malloc | test.cpp:266:12:266:12 | p |
nodes
| test.cpp:16:11:16:21 | mk_string_t indirection [string] | semmle.label | mk_string_t indirection [string] |
| test.cpp:18:5:18:30 | ... = ... | semmle.label | ... = ... |
@@ -116,16 +110,13 @@ nodes
| test.cpp:214:24:214:24 | p | semmle.label | p |
| test.cpp:216:10:216:10 | p | semmle.label | p |
| test.cpp:220:27:220:54 | call to malloc | semmle.label | call to malloc |
| test.cpp:220:43:220:48 | call to malloc | semmle.label | call to malloc |
| test.cpp:222:15:222:20 | buffer | semmle.label | buffer |
| test.cpp:228:27:228:54 | call to malloc | semmle.label | call to malloc |
| test.cpp:228:43:228:48 | call to malloc | semmle.label | call to malloc |
| test.cpp:232:10:232:15 | buffer | semmle.label | buffer |
| test.cpp:235:40:235:45 | buffer | semmle.label | buffer |
| test.cpp:236:5:236:26 | ... = ... | semmle.label | ... = ... |
| test.cpp:236:12:236:17 | p_str indirection [post update] [string] | semmle.label | p_str indirection [post update] [string] |
| test.cpp:241:20:241:38 | call to malloc | semmle.label | call to malloc |
| test.cpp:241:27:241:32 | call to malloc | semmle.label | call to malloc |
| test.cpp:242:16:242:19 | set_string output argument [string] | semmle.label | set_string output argument [string] |
| test.cpp:242:22:242:27 | buffer | semmle.label | buffer |
| test.cpp:243:12:243:14 | str indirection [string] | semmle.label | str indirection [string] |
@@ -133,12 +124,9 @@ nodes
| test.cpp:249:14:249:33 | call to my_alloc | semmle.label | call to my_alloc |
| test.cpp:250:12:250:12 | p | semmle.label | p |
| test.cpp:256:9:256:25 | call to malloc | semmle.label | call to malloc |
| test.cpp:256:17:256:22 | call to malloc | semmle.label | call to malloc |
| test.cpp:257:12:257:12 | p | semmle.label | p |
| test.cpp:262:15:262:30 | call to malloc | semmle.label | call to malloc |
| test.cpp:262:22:262:27 | call to malloc | semmle.label | call to malloc |
| test.cpp:264:13:264:30 | call to malloc | semmle.label | call to malloc |
| test.cpp:264:20:264:25 | call to malloc | semmle.label | call to malloc |
| test.cpp:266:12:266:12 | p | semmle.label | p |
subpaths
| test.cpp:242:22:242:27 | buffer | test.cpp:235:40:235:45 | buffer | test.cpp:236:12:236:17 | p_str indirection [post update] [string] | test.cpp:242:16:242:19 | set_string output argument [string] |
@@ -159,7 +147,5 @@ subpaths
| test.cpp:203:9:203:15 | call to strncpy | test.cpp:147:19:147:24 | call to malloc | test.cpp:203:22:203:27 | string | This write may overflow $@ by 2 elements. | test.cpp:203:22:203:27 | string | string |
| test.cpp:207:9:207:15 | call to strncpy | test.cpp:147:19:147:24 | call to malloc | test.cpp:207:22:207:27 | string | This write may overflow $@ by 3 elements. | test.cpp:207:22:207:27 | string | string |
| test.cpp:243:5:243:10 | call to memset | test.cpp:241:20:241:38 | call to malloc | test.cpp:243:12:243:21 | string | This write may overflow $@ by 1 element. | test.cpp:243:16:243:21 | string | string |
| test.cpp:243:5:243:10 | call to memset | test.cpp:241:27:241:32 | call to malloc | test.cpp:243:12:243:21 | string | This write may overflow $@ by 1 element. | test.cpp:243:16:243:21 | string | string |
| test.cpp:250:5:250:10 | call to memset | test.cpp:249:14:249:33 | call to my_alloc | test.cpp:250:12:250:12 | p | This write may overflow $@ by 1 element. | test.cpp:250:12:250:12 | p | p |
| test.cpp:266:5:266:10 | call to memset | test.cpp:262:15:262:30 | call to malloc | test.cpp:266:12:266:12 | p | This write may overflow $@ by 1 element. | test.cpp:266:12:266:12 | p | p |
| test.cpp:266:5:266:10 | call to memset | test.cpp:262:22:262:27 | call to malloc | test.cpp:266:12:266:12 | p | This write may overflow $@ by 1 element. | test.cpp:266:12:266:12 | p | p |

85
cpp/ql/test/query-tests/Security/CWE/CWE-193/InvalidPointerDeref.expected
View File

@@ -1,9 +1,9 @@
edges
| test.cpp:4:15:4:20 | call to malloc | test.cpp:5:15:5:22 | ... + ... |
| test.cpp:4:15:4:20 | call to malloc | test.cpp:5:15:5:22 | ... + ... |
| test.cpp:4:15:4:20 | call to malloc | test.cpp:6:14:6:15 | * ... |
| test.cpp:4:15:4:20 | call to malloc | test.cpp:6:14:6:15 | * ... |
| test.cpp:4:15:4:20 | call to malloc | test.cpp:8:14:8:21 | * ... |
| test.cpp:4:15:4:33 | call to malloc | test.cpp:5:15:5:22 | ... + ... |
| test.cpp:4:15:4:33 | call to malloc | test.cpp:5:15:5:22 | ... + ... |
| test.cpp:4:15:4:33 | call to malloc | test.cpp:6:14:6:15 | * ... |
| test.cpp:4:15:4:33 | call to malloc | test.cpp:6:14:6:15 | * ... |
| test.cpp:4:15:4:33 | call to malloc | test.cpp:8:14:8:21 | * ... |
| test.cpp:5:15:5:22 | ... + ... | test.cpp:5:15:5:22 | ... + ... |
| test.cpp:5:15:5:22 | ... + ... | test.cpp:6:14:6:15 | * ... |
| test.cpp:5:15:5:22 | ... + ... | test.cpp:6:14:6:15 | * ... |
@@ -12,12 +12,12 @@ edges
| test.cpp:5:15:5:22 | ... + ... | test.cpp:8:14:8:21 | * ... |
| test.cpp:5:15:5:22 | ... + ... | test.cpp:8:14:8:21 | * ... |
| test.cpp:6:14:6:15 | * ... | test.cpp:8:14:8:21 | * ... |
| test.cpp:16:15:16:20 | call to malloc | test.cpp:20:14:20:21 | * ... |
| test.cpp:28:15:28:20 | call to malloc | test.cpp:29:15:29:28 | ... + ... |
| test.cpp:28:15:28:20 | call to malloc | test.cpp:29:15:29:28 | ... + ... |
| test.cpp:28:15:28:20 | call to malloc | test.cpp:30:14:30:15 | * ... |
| test.cpp:28:15:28:20 | call to malloc | test.cpp:30:14:30:15 | * ... |
| test.cpp:28:15:28:20 | call to malloc | test.cpp:32:14:32:21 | * ... |
| test.cpp:16:15:16:33 | call to malloc | test.cpp:20:14:20:21 | * ... |
| test.cpp:28:15:28:37 | call to malloc | test.cpp:29:15:29:28 | ... + ... |
| test.cpp:28:15:28:37 | call to malloc | test.cpp:29:15:29:28 | ... + ... |
| test.cpp:28:15:28:37 | call to malloc | test.cpp:30:14:30:15 | * ... |
| test.cpp:28:15:28:37 | call to malloc | test.cpp:30:14:30:15 | * ... |
| test.cpp:28:15:28:37 | call to malloc | test.cpp:32:14:32:21 | * ... |
| test.cpp:29:15:29:28 | ... + ... | test.cpp:29:15:29:28 | ... + ... |
| test.cpp:29:15:29:28 | ... + ... | test.cpp:30:14:30:15 | * ... |
| test.cpp:29:15:29:28 | ... + ... | test.cpp:30:14:30:15 | * ... |
@@ -26,22 +26,9 @@ edges
| test.cpp:29:15:29:28 | ... + ... | test.cpp:32:14:32:21 | * ... |
| test.cpp:29:15:29:28 | ... + ... | test.cpp:32:14:32:21 | * ... |
| test.cpp:30:14:30:15 | * ... | test.cpp:32:14:32:21 | * ... |
| test.cpp:40:15:40:20 | call to malloc | test.cpp:41:15:41:28 | ... + ... |
| test.cpp:40:15:40:20 | call to malloc | test.cpp:41:15:41:28 | ... + ... |
| test.cpp:40:15:40:20 | call to malloc | test.cpp:42:14:42:15 | * ... |
| test.cpp:40:15:40:20 | call to malloc | test.cpp:42:14:42:15 | * ... |
| test.cpp:40:15:40:20 | call to malloc | test.cpp:44:14:44:21 | * ... |
| test.cpp:41:15:41:28 | ... + ... | test.cpp:41:15:41:28 | ... + ... |
| test.cpp:41:15:41:28 | ... + ... | test.cpp:42:14:42:15 | * ... |
| test.cpp:41:15:41:28 | ... + ... | test.cpp:42:14:42:15 | * ... |
| test.cpp:41:15:41:28 | ... + ... | test.cpp:42:14:42:15 | * ... |
| test.cpp:41:15:41:28 | ... + ... | test.cpp:42:14:42:15 | * ... |
| test.cpp:41:15:41:28 | ... + ... | test.cpp:44:14:44:21 | * ... |
| test.cpp:41:15:41:28 | ... + ... | test.cpp:44:14:44:21 | * ... |
| test.cpp:42:14:42:15 | * ... | test.cpp:44:14:44:21 | * ... |
| test.cpp:51:33:51:35 | end | test.cpp:60:34:60:37 | mk_array output argument |
| test.cpp:52:19:52:24 | call to malloc | test.cpp:53:5:53:23 | ... = ... |
| test.cpp:52:19:52:24 | call to malloc | test.cpp:53:12:53:23 | ... + ... |
| test.cpp:52:19:52:37 | call to malloc | test.cpp:53:5:53:23 | ... = ... |
| test.cpp:52:19:52:37 | call to malloc | test.cpp:53:12:53:23 | ... + ... |
| test.cpp:53:5:53:23 | ... = ... | test.cpp:51:33:51:35 | end |
| test.cpp:53:12:53:23 | ... + ... | test.cpp:53:5:53:23 | ... = ... |
| test.cpp:60:34:60:37 | mk_array output argument | test.cpp:67:9:67:14 | ... = ... |
@@ -183,8 +170,8 @@ edges
| test.cpp:781:14:781:27 | new[] | test.cpp:786:18:786:27 | access to array |
| test.cpp:792:60:792:62 | end | test.cpp:800:40:800:43 | mk_array_no_field_flow output argument |
| test.cpp:792:60:792:62 | end | test.cpp:832:40:832:43 | mk_array_no_field_flow output argument |
| test.cpp:793:14:793:19 | call to malloc | test.cpp:794:5:794:24 | ... = ... |
| test.cpp:793:14:793:19 | call to malloc | test.cpp:794:12:794:24 | ... + ... |
| test.cpp:793:14:793:32 | call to malloc | test.cpp:794:5:794:24 | ... = ... |
| test.cpp:793:14:793:32 | call to malloc | test.cpp:794:12:794:24 | ... + ... |
| test.cpp:794:5:794:24 | ... = ... | test.cpp:792:60:792:62 | end |
| test.cpp:794:12:794:24 | ... + ... | test.cpp:794:5:794:24 | ... = ... |
| test.cpp:800:40:800:43 | mk_array_no_field_flow output argument | test.cpp:807:7:807:12 | ... = ... |
@@ -193,29 +180,25 @@ edges
| test.cpp:815:52:815:54 | end | test.cpp:821:7:821:12 | ... = ... |
| test.cpp:832:40:832:43 | mk_array_no_field_flow output argument | test.cpp:833:37:833:39 | end |
| test.cpp:833:37:833:39 | end | test.cpp:815:52:815:54 | end |
| test.cpp:841:18:841:35 | call to malloc | test.cpp:842:3:842:20 | ... = ... |
| test.cpp:848:20:848:37 | call to malloc | test.cpp:849:5:849:22 | ... = ... |
nodes
| test.cpp:4:15:4:20 | call to malloc | semmle.label | call to malloc |
| test.cpp:4:15:4:33 | call to malloc | semmle.label | call to malloc |
| test.cpp:5:15:5:22 | ... + ... | semmle.label | ... + ... |
| test.cpp:5:15:5:22 | ... + ... | semmle.label | ... + ... |
| test.cpp:6:14:6:15 | * ... | semmle.label | * ... |
| test.cpp:6:14:6:15 | * ... | semmle.label | * ... |
| test.cpp:8:14:8:21 | * ... | semmle.label | * ... |
| test.cpp:16:15:16:20 | call to malloc | semmle.label | call to malloc |
| test.cpp:16:15:16:33 | call to malloc | semmle.label | call to malloc |
| test.cpp:20:14:20:21 | * ... | semmle.label | * ... |
| test.cpp:28:15:28:20 | call to malloc | semmle.label | call to malloc |
| test.cpp:28:15:28:37 | call to malloc | semmle.label | call to malloc |
| test.cpp:29:15:29:28 | ... + ... | semmle.label | ... + ... |
| test.cpp:29:15:29:28 | ... + ... | semmle.label | ... + ... |
| test.cpp:30:14:30:15 | * ... | semmle.label | * ... |
| test.cpp:30:14:30:15 | * ... | semmle.label | * ... |
| test.cpp:32:14:32:21 | * ... | semmle.label | * ... |
| test.cpp:40:15:40:20 | call to malloc | semmle.label | call to malloc |
| test.cpp:41:15:41:28 | ... + ... | semmle.label | ... + ... |
| test.cpp:41:15:41:28 | ... + ... | semmle.label | ... + ... |
| test.cpp:42:14:42:15 | * ... | semmle.label | * ... |
| test.cpp:42:14:42:15 | * ... | semmle.label | * ... |
| test.cpp:44:14:44:21 | * ... | semmle.label | * ... |
| test.cpp:51:33:51:35 | end | semmle.label | end |
| test.cpp:52:19:52:24 | call to malloc | semmle.label | call to malloc |
| test.cpp:52:19:52:37 | call to malloc | semmle.label | call to malloc |
| test.cpp:53:5:53:23 | ... = ... | semmle.label | ... = ... |
| test.cpp:53:12:53:23 | ... + ... | semmle.label | ... + ... |
| test.cpp:60:34:60:37 | mk_array output argument | semmle.label | mk_array output argument |
@@ -313,7 +296,7 @@ nodes
| test.cpp:781:14:781:27 | new[] | semmle.label | new[] |
| test.cpp:786:18:786:27 | access to array | semmle.label | access to array |
| test.cpp:792:60:792:62 | end | semmle.label | end |
| test.cpp:793:14:793:19 | call to malloc | semmle.label | call to malloc |
| test.cpp:793:14:793:32 | call to malloc | semmle.label | call to malloc |
| test.cpp:794:5:794:24 | ... = ... | semmle.label | ... = ... |
| test.cpp:794:12:794:24 | ... + ... | semmle.label | ... + ... |
| test.cpp:800:40:800:43 | mk_array_no_field_flow output argument | semmle.label | mk_array_no_field_flow output argument |
@@ -323,16 +306,18 @@ nodes
| test.cpp:821:7:821:12 | ... = ... | semmle.label | ... = ... |
| test.cpp:832:40:832:43 | mk_array_no_field_flow output argument | semmle.label | mk_array_no_field_flow output argument |
| test.cpp:833:37:833:39 | end | semmle.label | end |
| test.cpp:841:18:841:35 | call to malloc | semmle.label | call to malloc |
| test.cpp:842:3:842:20 | ... = ... | semmle.label | ... = ... |
| test.cpp:848:20:848:37 | call to malloc | semmle.label | call to malloc |
| test.cpp:849:5:849:22 | ... = ... | semmle.label | ... = ... |
subpaths
#select
| test.cpp:6:14:6:15 | * ... | test.cpp:4:15:4:20 | call to malloc | test.cpp:6:14:6:15 | * ... | This read might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:4:15:4:20 | call to malloc | call to malloc | test.cpp:5:19:5:22 | size | size |
| test.cpp:8:14:8:21 | * ... | test.cpp:4:15:4:20 | call to malloc | test.cpp:8:14:8:21 | * ... | This read might be out of bounds, as the pointer might be equal to $@ + $@ + 1. | test.cpp:4:15:4:20 | call to malloc | call to malloc | test.cpp:5:19:5:22 | size | size |
| test.cpp:20:14:20:21 | * ... | test.cpp:16:15:16:20 | call to malloc | test.cpp:20:14:20:21 | * ... | This read might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:16:15:16:20 | call to malloc | call to malloc | test.cpp:17:19:17:22 | size | size |
| test.cpp:30:14:30:15 | * ... | test.cpp:28:15:28:20 | call to malloc | test.cpp:30:14:30:15 | * ... | This read might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:28:15:28:20 | call to malloc | call to malloc | test.cpp:29:20:29:27 | ... + ... | ... + ... |
| test.cpp:32:14:32:21 | * ... | test.cpp:28:15:28:20 | call to malloc | test.cpp:32:14:32:21 | * ... | This read might be out of bounds, as the pointer might be equal to $@ + $@ + 1. | test.cpp:28:15:28:20 | call to malloc | call to malloc | test.cpp:29:20:29:27 | ... + ... | ... + ... |
| test.cpp:42:14:42:15 | * ... | test.cpp:40:15:40:20 | call to malloc | test.cpp:42:14:42:15 | * ... | This read might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:40:15:40:20 | call to malloc | call to malloc | test.cpp:41:20:41:27 | ... - ... | ... - ... |
| test.cpp:44:14:44:21 | * ... | test.cpp:40:15:40:20 | call to malloc | test.cpp:44:14:44:21 | * ... | This read might be out of bounds, as the pointer might be equal to $@ + $@ + 1. | test.cpp:40:15:40:20 | call to malloc | call to malloc | test.cpp:41:20:41:27 | ... - ... | ... - ... |
| test.cpp:67:9:67:14 | ... = ... | test.cpp:52:19:52:24 | call to malloc | test.cpp:67:9:67:14 | ... = ... | This write might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:52:19:52:24 | call to malloc | call to malloc | test.cpp:53:20:53:23 | size | size |
| test.cpp:6:14:6:15 | * ... | test.cpp:4:15:4:33 | call to malloc | test.cpp:6:14:6:15 | * ... | This read might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:4:15:4:33 | call to malloc | call to malloc | test.cpp:5:19:5:22 | size | size |
| test.cpp:8:14:8:21 | * ... | test.cpp:4:15:4:33 | call to malloc | test.cpp:8:14:8:21 | * ... | This read might be out of bounds, as the pointer might be equal to $@ + $@ + 1. | test.cpp:4:15:4:33 | call to malloc | call to malloc | test.cpp:5:19:5:22 | size | size |
| test.cpp:20:14:20:21 | * ... | test.cpp:16:15:16:33 | call to malloc | test.cpp:20:14:20:21 | * ... | This read might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:16:15:16:33 | call to malloc | call to malloc | test.cpp:17:19:17:22 | size | size |
| test.cpp:30:14:30:15 | * ... | test.cpp:28:15:28:37 | call to malloc | test.cpp:30:14:30:15 | * ... | This read might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:28:15:28:37 | call to malloc | call to malloc | test.cpp:29:20:29:27 | ... + ... | ... + ... |
| test.cpp:32:14:32:21 | * ... | test.cpp:28:15:28:37 | call to malloc | test.cpp:32:14:32:21 | * ... | This read might be out of bounds, as the pointer might be equal to $@ + $@ + 1. | test.cpp:28:15:28:37 | call to malloc | call to malloc | test.cpp:29:20:29:27 | ... + ... | ... + ... |
| test.cpp:67:9:67:14 | ... = ... | test.cpp:52:19:52:37 | call to malloc | test.cpp:67:9:67:14 | ... = ... | This write might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:52:19:52:37 | call to malloc | call to malloc | test.cpp:53:20:53:23 | size | size |
| test.cpp:201:5:201:19 | ... = ... | test.cpp:194:15:194:33 | call to malloc | test.cpp:201:5:201:19 | ... = ... | This write might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:194:15:194:33 | call to malloc | call to malloc | test.cpp:195:21:195:23 | len | len |
| test.cpp:213:5:213:13 | ... = ... | test.cpp:205:15:205:33 | call to malloc | test.cpp:213:5:213:13 | ... = ... | This write might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:205:15:205:33 | call to malloc | call to malloc | test.cpp:206:21:206:23 | len | len |
| test.cpp:232:3:232:20 | ... = ... | test.cpp:231:18:231:30 | new[] | test.cpp:232:3:232:20 | ... = ... | This write might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:231:18:231:30 | new[] | new[] | test.cpp:232:11:232:15 | index | index |
@@ -359,5 +344,7 @@ subpaths
| test.cpp:772:16:772:29 | access to array | test.cpp:754:18:754:31 | new[] | test.cpp:772:16:772:29 | access to array | This read might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:754:18:754:31 | new[] | new[] | test.cpp:767:22:767:28 | ... + ... | ... + ... |
| test.cpp:772:16:772:29 | access to array | test.cpp:754:18:754:31 | new[] | test.cpp:772:16:772:29 | access to array | This read might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:754:18:754:31 | new[] | new[] | test.cpp:772:22:772:28 | ... + ... | ... + ... |
| test.cpp:786:18:786:27 | access to array | test.cpp:781:14:781:27 | new[] | test.cpp:786:18:786:27 | access to array | This read might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:781:14:781:27 | new[] | new[] | test.cpp:786:20:786:26 | ... + ... | ... + ... |
| test.cpp:807:7:807:12 | ... = ... | test.cpp:793:14:793:19 | call to malloc | test.cpp:807:7:807:12 | ... = ... | This write might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:793:14:793:19 | call to malloc | call to malloc | test.cpp:794:21:794:24 | size | size |
| test.cpp:821:7:821:12 | ... = ... | test.cpp:793:14:793:19 | call to malloc | test.cpp:821:7:821:12 | ... = ... | This write might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:793:14:793:19 | call to malloc | call to malloc | test.cpp:794:21:794:24 | size | size |
| test.cpp:807:7:807:12 | ... = ... | test.cpp:793:14:793:32 | call to malloc | test.cpp:807:7:807:12 | ... = ... | This write might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:793:14:793:32 | call to malloc | call to malloc | test.cpp:794:21:794:24 | size | size |
| test.cpp:821:7:821:12 | ... = ... | test.cpp:793:14:793:32 | call to malloc | test.cpp:821:7:821:12 | ... = ... | This write might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:793:14:793:32 | call to malloc | call to malloc | test.cpp:794:21:794:24 | size | size |
| test.cpp:842:3:842:20 | ... = ... | test.cpp:841:18:841:35 | call to malloc | test.cpp:842:3:842:20 | ... = ... | This write might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:841:18:841:35 | call to malloc | call to malloc | test.cpp:842:11:842:15 | index | index |
| test.cpp:849:5:849:22 | ... = ... | test.cpp:848:20:848:37 | call to malloc | test.cpp:849:5:849:22 | ... = ... | This write might be out of bounds, as the pointer might be equal to $@ + $@. | test.cpp:848:20:848:37 | call to malloc | call to malloc | test.cpp:849:13:849:17 | index | index |

53
cpp/ql/test/query-tests/Security/CWE/CWE-193/test.cpp
View File

@@ -1,7 +1,7 @@
char *malloc(int size);
using size_t = decltype(sizeof 0); void* malloc(size_t size);
void test1(int size) {
char* p = malloc(size);
char* p = (char*)malloc(size);
char* q = p + size; // $ alloc=L4
char a = *q; // $ deref=L6 // BAD
char b = *(q - 1); // GOOD
@@ -13,7 +13,7 @@ void test1(int size) {
}
void test2(int size) {
char* p = malloc(size);
char* p = (char*)malloc(size);
char* q = p + size - 1; // $ alloc=L16
char a = *q; // GOOD
char b = *(q - 1); // GOOD
@@ -25,7 +25,7 @@ void test2(int size) {
}
void test3(int size) {
char* p = malloc(size + 1);
char* p = (char*)malloc(size + 1);
char* q = p + (size + 1); // $ alloc=L28+1
char a = *q; // $ deref=L30 // BAD
char b = *(q - 1); // GOOD
@@ -37,11 +37,11 @@ void test3(int size) {
}
void test4(int size) {
char* p = malloc(size - 1);
char* q = p + (size - 1); // $ alloc=L40-1
char a = *q; // $ deref=L42 // BAD
char* p = (char*)malloc(size - 1);
char* q = p + (size - 1); // $ MISSING: alloc=L40-1
char a = *q; // $ MISSING: deref=L42 // BAD [NOT DETECTED]
char b = *(q - 1); // GOOD
char c = *(q + 1); // $ deref=L44+1 // BAD
char c = *(q + 1); // $ MISSING: deref=L44+1 // BAD [NOT DETECTED]
char d = *(q + size); // BAD [NOT DETECTED]
char e = *(q - size); // GOOD
char f = *(q + size + 1); // BAD [NOT DETECTED]
@@ -49,7 +49,7 @@ void test4(int size) {
}
char* mk_array(int size, char** end) {
char* begin = malloc(size);
char* begin = (char*)malloc(size);
*end = begin + size; // $ alloc=L52
return begin;
@@ -79,7 +79,7 @@ struct array_t {
array_t mk_array(int size) {
array_t arr;
arr.begin = malloc(size);
arr.begin = (char*)malloc(size);
arr.end = arr.begin + size; // $ MISSING: alloc=L82
return arr;
@@ -121,7 +121,7 @@ void test7(int size) {
void test8(int size) {
array_t arr;
char* p = malloc(size);
char* p = (char*)malloc(size);
arr.begin = p;
arr.end = p + size; // $ alloc=L124
@@ -140,7 +140,7 @@ void test8(int size) {
array_t *mk_array_p(int size) {
array_t *arr = (array_t*) malloc(sizeof(array_t));
arr->begin = malloc(size);
arr->begin = (char*)malloc(size);
arr->end = arr->begin + size; // $ MISSING: alloc=L143
return arr;
@@ -185,7 +185,7 @@ void deref_plus_one(char* q) {
}
void test11(unsigned size) {
char *p = malloc(size);
char *p = (char*)malloc(size);
char *q = p + size - 1; // $ alloc=L188
deref_plus_one(q);
}
@@ -215,7 +215,7 @@ void test13(unsigned len, unsigned index) {
bool unknown();
void test14(unsigned long n, char *p) {
void test14(size_t n, char *p) {
while (unknown()) {
n++;
p = (char *)malloc(n);
@@ -706,7 +706,7 @@ void deref(char* q) {
char x = *q; // $ deref=L714->L705->L706 // BAD
}
void test35(unsigned long size, char* q)
void test35(size_t size, char* q)
{
char* p = new char[size];
char* end = p + size; // $ alloc=L711
@@ -734,10 +734,10 @@ void test36(unsigned size, unsigned n) {
}
}
void test37(unsigned long n)
void test37(size_t n)
{
int *p = new int[n];
for (unsigned long i = n; i != 0u; i--)
for (size_t i = n; i != 0u; i--)
{
p[n - i] = 0; // GOOD
}
@@ -790,7 +790,7 @@ void test38_simple(unsigned size, unsigned pos, unsigned numParams) {
}
void mk_array_no_field_flow(int size, char** begin, char** end) {
*begin = malloc(size);
*begin = (char*)malloc(size);
*end = *begin + size; // $ alloc=L793
}
@@ -832,3 +832,20 @@ void test7_no_field_flow(int size) {
mk_array_no_field_flow(size, &begin, &end);
test7_callee_no_field_flow(begin, end);
}
void test15_with_malloc(size_t index) {
size_t size = index + 13;
if(size < index) {
return;
}
int* newname = (int*)malloc(size);
newname[index] = 0; // $ SPURIOUS: alloc=L841 deref=L842 // GOOD [FALSE POSITIVE]
}
void test16_with_malloc(size_t index) {
size_t size = index + 13;
if(size >= index) {
int* newname = (int*)malloc(size);
newname[index] = 0; // $ SPURIOUS: alloc=L848 deref=L849 // GOOD [FALSE POSITIVE]
}
}

21
csharp/CSharp.sln
View File

@@ -1,5 +1,4 @@

Microsoft Visual Studio Solution File, Format Version 12.00
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 15
VisualStudioVersion = 15.0.27130.2036
MinimumVisualStudioVersion = 10.0.40219.1
@@ -15,6 +14,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.De
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.Standalone", "extractor\Semmle.Extraction.CSharp.Standalone\Semmle.Extraction.CSharp.Standalone.csproj", "{D00E7D25-0FA0-48EC-B048-CD60CE1B30D8}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.StubGenerator", "extractor\Semmle.Extraction.CSharp.StubGenerator\Semmle.Extraction.CSharp.StubGenerator.csproj", "{B7C9FD47-A78C-4C20-AC29-B0AE638ADE9D}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.Util", "extractor\Semmle.Extraction.CSharp.Util\Semmle.Extraction.CSharp.Util.csproj", "{998A0D4C-8BFC-4513-A28D-4816AFB89882}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CIL.Driver", "extractor\Semmle.Extraction.CIL.Driver\Semmle.Extraction.CIL.Driver.csproj", "{EFA400B3-C1CE-446F-A4E2-8B44E61EF47C}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.Driver", "extractor\Semmle.Extraction.CSharp.Driver\Semmle.Extraction.CSharp.Driver.csproj", "{C36453BF-0C82-448A-B15D-26947503A2D3}"
@@ -29,6 +32,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.CSharp", "
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.CSharp.Tests", "autobuilder\Semmle.Autobuild.CSharp.Tests\Semmle.Autobuild.CSharp.Tests.csproj", "{34256E8F-866A-46C1-800E-3DF69FD1DCB7}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Extraction.CSharp.DependencyStubGenerator", "extractor\Semmle.Extraction.CSharp.DependencyStubGenerator\Semmle.Extraction.CSharp.DependencyStubGenerator.csproj", "{0EDA21A3-ADD8-4C10-B494-58B12B526B76}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -85,6 +90,18 @@ Global
{34256E8F-866A-46C1-800E-3DF69FD1DCB7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{34256E8F-866A-46C1-800E-3DF69FD1DCB7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{34256E8F-866A-46C1-800E-3DF69FD1DCB7}.Release|Any CPU.Build.0 = Release|Any CPU
{B7C9FD47-A78C-4C20-AC29-B0AE638ADE9D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B7C9FD47-A78C-4C20-AC29-B0AE638ADE9D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B7C9FD47-A78C-4C20-AC29-B0AE638ADE9D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B7C9FD47-A78C-4C20-AC29-B0AE638ADE9D}.Release|Any CPU.Build.0 = Release|Any CPU
{998A0D4C-8BFC-4513-A28D-4816AFB89882}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{998A0D4C-8BFC-4513-A28D-4816AFB89882}.Debug|Any CPU.Build.0 = Debug|Any CPU
{998A0D4C-8BFC-4513-A28D-4816AFB89882}.Release|Any CPU.ActiveCfg = Release|Any CPU
{998A0D4C-8BFC-4513-A28D-4816AFB89882}.Release|Any CPU.Build.0 = Release|Any CPU
{0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Debug|Any CPU.Build.0 = Debug|Any CPU
{0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Release|Any CPU.ActiveCfg = Release|Any CPU
{0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

2
csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs
View File

@@ -267,7 +267,7 @@ namespace Semmle.Autobuild.Shared
protected DiagnosticClassifier DiagnosticClassifier { get; }
private readonly ILogger logger = new ConsoleLogger(Verbosity.Info);
private readonly ILogger logger = new ConsoleLogger(Verbosity.Info, logThreadId: false);
private readonly IDiagnosticsWriter diagnostics;

36
csharp/autobuilder/Semmle.Autobuild.Shared/BuildActions.cs
View File

@@ -187,12 +187,12 @@ namespace Semmle.Autobuild.Shared
bool IBuildActions.FileExists(string file) => File.Exists(file);
private static ProcessStartInfo GetProcessStartInfo(string exe, string arguments, string? workingDirectory, IDictionary<string, string>? environment, bool redirectStandardOutput)
private static ProcessStartInfo GetProcessStartInfo(string exe, string arguments, string? workingDirectory, IDictionary<string, string>? environment)
{
var pi = new ProcessStartInfo(exe, arguments)
{
UseShellExecute = false,
RedirectStandardOutput = redirectStandardOutput
RedirectStandardOutput = true
};
if (workingDirectory is not null)
pi.WorkingDirectory = workingDirectory;
@@ -204,40 +204,22 @@ namespace Semmle.Autobuild.Shared
int IBuildActions.RunProcess(string exe, string args, string? workingDirectory, System.Collections.Generic.IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError)
{
var pi = GetProcessStartInfo(exe, args, workingDirectory, env, true);
using var p = new Process
{
StartInfo = pi
};
p.StartInfo.RedirectStandardError = true;
p.OutputDataReceived += new DataReceivedEventHandler((sender, e) => onOutput(e.Data));
p.ErrorDataReceived += new DataReceivedEventHandler((sender, e) => onError(e.Data));
var pi = GetProcessStartInfo(exe, args, workingDirectory, env);
pi.RedirectStandardError = true;
p.Start();
p.BeginErrorReadLine();
p.BeginOutputReadLine();
p.WaitForExit();
return p.ExitCode;
return pi.ReadOutput(out _, onOut: s => onOutput(s), onError: s => onError(s));
}
int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? environment)
{
var pi = GetProcessStartInfo(cmd, args, workingDirectory, environment, false);
using var p = Process.Start(pi);
if (p is null)
{
return -1;
}
p.WaitForExit();
return p.ExitCode;
var pi = GetProcessStartInfo(cmd, args, workingDirectory, environment);
return pi.ReadOutput(out _, onOut: Console.WriteLine, onError: null);
}
int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? environment, out IList<string> stdOut)
{
var pi = GetProcessStartInfo(cmd, args, workingDirectory, environment, true);
return pi.ReadOutput(out stdOut);
var pi = GetProcessStartInfo(cmd, args, workingDirectory, environment);
return pi.ReadOutput(out stdOut, onOut: null, onError: null);
}
void IBuildActions.DirectoryDelete(string dir, bool recursive) => Directory.Delete(dir, recursive);

2
csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs
View File

@@ -38,7 +38,7 @@ namespace Semmle.Extraction.CIL.Driver
}
var options = new ExtractorOptions(args);
using var logger = new ConsoleLogger(options.Verbosity);
using var logger = new ConsoleLogger(options.Verbosity, logThreadId: false);
var actions = options.AssembliesToExtract
.Select(asm => asm.Filename)

202
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -27,12 +27,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private int conflictedReferences = 0;
private readonly IDependencyOptions options;
private readonly DirectoryInfo sourceDir;
private readonly DotNet dotnet;
private readonly IDotNet dotnet;
private readonly FileContent fileContent;
private readonly TemporaryDirectory packageDirectory;
private TemporaryDirectory? razorWorkingDirectory;
private readonly Git git;
private readonly TemporaryDirectory tempWorkingDirectory;
/// <summary>
/// Performs C# dependency fetching.
@@ -49,7 +47,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
try
{
this.dotnet = new DotNet(options, progressMonitor);
this.dotnet = DotNet.Make(options, progressMonitor);
}
catch
{
@@ -60,20 +58,21 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
this.progressMonitor.FindingFiles(srcDir);
packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
var allFiles = GetAllFiles().ToList();
var smallFiles = allFiles.SelectSmallFiles(progressMonitor).SelectFileNames();
this.fileContent = new FileContent(progressMonitor, smallFiles);
this.allSources = allFiles.SelectFileNamesByExtension(".cs").ToList();
var allProjects = allFiles.SelectFileNamesByExtension(".csproj");
tempWorkingDirectory = new TemporaryDirectory(GetTemporaryWorkingDirectory());
var allFiles = GetAllFiles();
var binaryFileExtensions = new HashSet<string>(new[] { ".dll", ".exe" }); // TODO: add more binary file extensions.
var allNonBinaryFiles = allFiles.Where(f => !binaryFileExtensions.Contains(f.Extension.ToLowerInvariant())).ToList();
var smallNonBinaryFiles = allNonBinaryFiles.SelectSmallFiles(progressMonitor).SelectFileNames();
this.fileContent = new FileContent(progressMonitor, smallNonBinaryFiles);
this.allSources = allNonBinaryFiles.SelectFileNamesByExtension(".cs").ToList();
var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj");
var solutions = options.SolutionFile is not null
? new[] { options.SolutionFile }
: allFiles.SelectFileNamesByExtension(".sln");
// If DLL reference paths are specified on the command-line, use those to discover
// assemblies. Otherwise (the default), query the git CLI to determine which DLL files
// are tracked as part of the repository.
this.git = new Git(this.progressMonitor);
var dllDirNames = options.DllDirs.Count == 0 ? this.git.ListFiles("*.dll") : options.DllDirs.Select(Path.GetFullPath).ToList();
: allNonBinaryFiles.SelectFileNamesByExtension(".sln");
var dllDirNames = options.DllDirs.Count == 0
? allFiles.SelectFileNamesByExtension(".dll").ToList()
: options.DllDirs.Select(Path.GetFullPath).ToList();
// Find DLLs in the .Net / Asp.Net Framework
if (options.ScanNetFrameworkDlls)
@@ -106,7 +105,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var restoredProjects = RestoreSolutions(solutions);
var projects = allProjects.Except(restoredProjects);
RestoreProjects(projects);
DownloadMissingPackages(allFiles);
DownloadMissingPackages(allNonBinaryFiles);
}
assemblyCache = new AssemblyCache(dllDirNames, progressMonitor);
@@ -117,6 +116,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
UseReference(filename);
}
RemoveRuntimeNugetPackageReferences();
ResolveConflicts();
// Output the findings
@@ -134,9 +134,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) &&
shouldExtractWebViews)
{
GenerateSourceFilesFromWebViews(allFiles);
GenerateSourceFilesFromWebViews(allNonBinaryFiles);
}
GenerateSourceFileFromImplicitUsings();
progressMonitor.Summary(
AllSourceFiles.Count(),
ProjectSourceFiles.Count(),
@@ -149,6 +151,78 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
DateTime.Now - startTime);
}
private void RemoveRuntimeNugetPackageReferences()
{
if (!options.UseNuGet)
{
return;
}
var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant();
var runtimePackageNamePrefixes = new[]
{
Path.Combine(packageFolder, "microsoft.netcore.app.runtime"),
Path.Combine(packageFolder, "microsoft.aspnetcore.app.runtime"),
Path.Combine(packageFolder, "microsoft.windowsdesktop.app.runtime"),
// legacy runtime packages:
Path.Combine(packageFolder, "runtime.linux-x64.microsoft.netcore.app"),
Path.Combine(packageFolder, "runtime.osx-x64.microsoft.netcore.app"),
Path.Combine(packageFolder, "runtime.win-x64.microsoft.netcore.app"),
};
foreach (var filename in usedReferences.Keys)
{
var lowerFilename = filename.ToLowerInvariant();
if (runtimePackageNamePrefixes.Any(prefix => lowerFilename.StartsWith(prefix)))
{
usedReferences.Remove(filename);
progressMonitor.RemovedReference(filename);
}
}
}
private void GenerateSourceFileFromImplicitUsings()
{
var usings = new HashSet<string>();
if (!fileContent.UseImplicitUsings)
{
return;
}
// Hardcoded values from https://learn.microsoft.com/en-us/dotnet/core/project-sdk/overview#implicit-using-directives
usings.UnionWith(new[] { "System", "System.Collections.Generic", "System.IO", "System.Linq", "System.Net.Http", "System.Threading",
"System.Threading.Tasks" });
if (fileContent.UseAspNetDlls)
{
usings.UnionWith(new[] { "System.Net.Http.Json", "Microsoft.AspNetCore.Builder", "Microsoft.AspNetCore.Hosting",
"Microsoft.AspNetCore.Http", "Microsoft.AspNetCore.Routing", "Microsoft.Extensions.Configuration",
"Microsoft.Extensions.DependencyInjection", "Microsoft.Extensions.Hosting", "Microsoft.Extensions.Logging" });
}
usings.UnionWith(fileContent.CustomImplicitUsings);
if (usings.Count > 0)
{
var tempDir = GetTemporaryWorkingDirectory("implicitUsings");
var path = Path.Combine(tempDir, "GlobalUsings.g.cs");
using (var writer = new StreamWriter(path))
{
writer.WriteLine("// <auto-generated/>");
writer.WriteLine("");
foreach (var u in usings.OrderBy(u => u))
{
writer.WriteLine($"global using global::{u};");
}
}
this.allSources.Add(path);
}
}
private void GenerateSourceFilesFromWebViews(List<FileInfo> allFiles)
{
progressMonitor.LogInfo($"Generating source files from cshtml and razor files.");
@@ -165,8 +239,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
try
{
var razor = new Razor(sdk, dotnet, progressMonitor);
razorWorkingDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "razor"));
var generatedFiles = razor.GenerateFiles(views, usedReferences.Keys, razorWorkingDirectory.ToString());
var targetDir = GetTemporaryWorkingDirectory("razor");
var generatedFiles = razor.GenerateFiles(views, usedReferences.Keys, targetDir);
this.allSources.AddRange(generatedFiles);
}
catch (Exception ex)
@@ -178,18 +252,27 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
public DependencyManager(string srcDir) : this(srcDir, DependencyOptions.Default, new ConsoleLogger(Verbosity.Info)) { }
public DependencyManager(string srcDir) : this(srcDir, DependencyOptions.Default, new ConsoleLogger(Verbosity.Info, logThreadId: true)) { }
private IEnumerable<FileInfo> GetAllFiles() =>
sourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true })
.Where(d => d.Extension != ".dll" && !options.ExcludesFile(d.FullName));
private IEnumerable<FileInfo> GetAllFiles()
{
var files = sourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true })
.Where(d => !options.ExcludesFile(d.FullName));
if (options.DotNetPath != null)
{
files = files.Where(f => !f.FullName.StartsWith(options.DotNetPath, StringComparison.OrdinalIgnoreCase));
}
return files;
}
/// <summary>
/// Computes a unique temp directory for the packages associated
/// with this source tree. Use a SHA1 of the directory name.
/// </summary>
/// <returns>The full path of the temp directory.</returns>
private static string ComputeTempDirectory(string srcDir, string subfolderName = "packages")
private static string ComputeTempDirectory(string srcDir)
{
var bytes = Encoding.Unicode.GetBytes(srcDir);
var sha = SHA1.HashData(bytes);
@@ -197,7 +280,35 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
foreach (var b in sha.Take(8))
sb.AppendFormat("{0:x2}", b);
return Path.Combine(Path.GetTempPath(), "GitHub", subfolderName, sb.ToString());
return Path.Combine(Path.GetTempPath(), "GitHub", "packages", sb.ToString());
}
private static string GetTemporaryWorkingDirectory()
{
var tempFolder = EnvironmentVariables.GetScratchDirectory();
if (string.IsNullOrEmpty(tempFolder))
{
var tempPath = Path.GetTempPath();
var name = Guid.NewGuid().ToString("N").ToUpper();
tempFolder = Path.Combine(tempPath, "GitHub", name);
}
return tempFolder;
}
/// <summary>
/// Creates a temporary directory with the given subfolder name.
/// The created directory might be inside the repo folder, and it is deleted when the object is disposed.
/// </summary>
/// <param name="subfolder"></param>
/// <returns></returns>
private string GetTemporaryWorkingDirectory(string subfolder)
{
var temp = Path.Combine(tempWorkingDirectory.ToString(), subfolder);
Directory.CreateDirectory(temp);
return temp;
}
/// <summary>
@@ -352,8 +463,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
private bool RestoreProject(string project, out string stdout, string? pathToNugetConfig = null) =>
dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, out stdout, pathToNugetConfig);
private bool RestoreProject(string project, string? pathToNugetConfig = null) =>
dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, pathToNugetConfig);
private bool RestoreSolution(string solution, out IEnumerable<string> projects) =>
dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, out projects);
@@ -376,25 +487,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// <summary>
/// Executes `dotnet restore` on all projects in projects.
/// This is done in parallel for performance reasons.
/// To ensure that output is not interleaved, the output of each
/// restore is collected and printed.
/// </summary>
/// <param name="projects">A list of paths to project files.</param>
private void RestoreProjects(IEnumerable<string> projects)
{
var stdoutLines = projects
.AsParallel()
.WithDegreeOfParallelism(options.Threads)
.Select(project =>
{
RestoreProject(project, out var stdout);
return stdout;
})
.ToList();
foreach (var line in stdoutLines)
Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project =>
{
Console.WriteLine(line);
}
RestoreProject(project);
});
}
private void DownloadMissingPackages(List<FileInfo> allFiles)
@@ -421,30 +521,30 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName)
.Select(d => Path.GetFileName(d).ToLowerInvariant());
var notYetDownloadedPackages = fileContent.AllPackages.Except(alreadyDownloadedPackages);
foreach (var package in notYetDownloadedPackages)
Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package =>
{
progressMonitor.NugetInstall(package);
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package));
var success = dotnet.New(tempDir.DirInfo.FullName);
if (!success)
{
continue;
return;
}
success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
if (!success)
{
continue;
return;
}
success = RestoreProject(tempDir.DirInfo.FullName, out var stdout, nugetConfig);
Console.WriteLine(stdout);
success = RestoreProject(tempDir.DirInfo.FullName, nugetConfig);
// TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
if (!success)
{
progressMonitor.FailedToRestoreNugetPackage(package);
}
}
});
}
private void AnalyseSolutions(IEnumerable<string> solutions)
@@ -467,7 +567,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public void Dispose()
{
packageDirectory?.Dispose();
razorWorkingDirectory?.Dispose();
tempWorkingDirectory?.Dispose();
}
}
}

75
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
View File

@@ -1,10 +1,8 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using Semmle.Util;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
@@ -13,85 +11,51 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// </summary>
internal partial class DotNet : IDotNet
{
private readonly IDotNetCliInvoker dotnetCliInvoker;
private readonly ProgressMonitor progressMonitor;
private readonly string dotnet;
public DotNet(IDependencyOptions options, ProgressMonitor progressMonitor)
private DotNet(IDotNetCliInvoker dotnetCliInvoker, ProgressMonitor progressMonitor)
{
this.progressMonitor = progressMonitor;
this.dotnet = Path.Combine(options.DotNetPath ?? string.Empty, "dotnet");
this.dotnetCliInvoker = dotnetCliInvoker;
Info();
}
private DotNet(IDependencyOptions options, ProgressMonitor progressMonitor) : this(new DotNetCliInvoker(progressMonitor, Path.Combine(options.DotNetPath ?? string.Empty, "dotnet")), progressMonitor) { }
internal static IDotNet Make(IDotNetCliInvoker dotnetCliInvoker, ProgressMonitor progressMonitor) => new DotNet(dotnetCliInvoker, progressMonitor);
public static IDotNet Make(IDependencyOptions options, ProgressMonitor progressMonitor) => new DotNet(options, progressMonitor);
private void Info()
{
// TODO: make sure the below `dotnet` version is matching the one specified in global.json
var res = RunCommand("--info");
var res = dotnetCliInvoker.RunCommand("--info");
if (!res)
{
throw new Exception($"{dotnet} --info failed.");
throw new Exception($"{dotnetCliInvoker.Exec} --info failed.");
}
}
private ProcessStartInfo MakeDotnetStartInfo(string args, bool redirectStandardOutput)
{
var startInfo = new ProcessStartInfo(dotnet, args)
{
UseShellExecute = false,
RedirectStandardOutput = redirectStandardOutput
};
// Set the .NET CLI language to English to avoid localized output.
startInfo.EnvironmentVariables["DOTNET_CLI_UI_LANGUAGE"] = "en";
return startInfo;
}
private bool RunCommand(string args)
{
progressMonitor.RunningProcess($"{dotnet} {args}");
using var proc = Process.Start(MakeDotnetStartInfo(args, redirectStandardOutput: false));
proc?.WaitForExit();
var exitCode = proc?.ExitCode ?? -1;
if (exitCode != 0)
{
progressMonitor.CommandFailed(dotnet, args, exitCode);
return false;
}
return true;
}
private bool RunCommand(string args, out IList<string> output)
{
progressMonitor.RunningProcess($"{dotnet} {args}");
var pi = MakeDotnetStartInfo(args, redirectStandardOutput: true);
var exitCode = pi.ReadOutput(out output);
if (exitCode != 0)
{
progressMonitor.CommandFailed(dotnet, args, exitCode);
return false;
}
return true;
}
private static string GetRestoreArgs(string projectOrSolutionFile, string packageDirectory) =>
$"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true";
public bool RestoreProjectToDirectory(string projectFile, string packageDirectory, out string stdout, string? pathToNugetConfig = null)
public bool RestoreProjectToDirectory(string projectFile, string packageDirectory, string? pathToNugetConfig = null)
{
var args = GetRestoreArgs(projectFile, packageDirectory);
if (pathToNugetConfig != null)
{
args += $" --configfile \"{pathToNugetConfig}\"";
}
var success = RunCommand(args, out var output);
stdout = string.Join("\n", output);
return success;
return dotnetCliInvoker.RunCommand(args);
}
public bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, out IEnumerable<string> projects)
{
var args = GetRestoreArgs(solutionFile, packageDirectory);
args += " --verbosity normal";
if (RunCommand(args, out var output))
if (dotnetCliInvoker.RunCommand(args, out var output))
{
var regex = RestoreProjectRegex();
projects = output
@@ -108,13 +72,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public bool New(string folder)
{
var args = $"new console --no-restore --output \"{folder}\"";
return RunCommand(args);
return dotnetCliInvoker.RunCommand(args);
}
public bool AddPackage(string folder, string package)
{
var args = $"add \"{folder}\" package \"{package}\" --no-restore";
return RunCommand(args);
return dotnetCliInvoker.RunCommand(args);
}
public IList<string> GetListedRuntimes() => GetListed("--list-runtimes", "runtime");
@@ -123,9 +87,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private IList<string> GetListed(string args, string artifact)
{
if (RunCommand(args, out var artifacts))
if (dotnetCliInvoker.RunCommand(args, out var artifacts))
{
progressMonitor.LogInfo($"Found {artifact}s: {string.Join("\n", artifacts)}");
return artifacts;
}
return new List<string>();
@@ -134,7 +97,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public bool Exec(string execArgs)
{
var args = $"exec {execArgs}";
return RunCommand(args);
return dotnetCliInvoker.RunCommand(args);
}
[GeneratedRegex("Restored\\s+(.+\\.csproj)", RegexOptions.Compiled)]

64
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs Normal file
View File

@@ -0,0 +1,64 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using Semmle.Util;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
/// <summary>
/// Low level utilities to run the "dotnet" command.
/// </summary>
internal sealed class DotNetCliInvoker : IDotNetCliInvoker
{
private readonly ProgressMonitor progressMonitor;
public string Exec { get; }
public DotNetCliInvoker(ProgressMonitor progressMonitor, string exec)
{
this.progressMonitor = progressMonitor;
this.Exec = exec;
}
private ProcessStartInfo MakeDotnetStartInfo(string args)
{
var startInfo = new ProcessStartInfo(Exec, args)
{
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true
};
// Set the .NET CLI language to English to avoid localized output.
startInfo.EnvironmentVariables["DOTNET_CLI_UI_LANGUAGE"] = "en";
return startInfo;
}
private bool RunCommandAux(string args, out IList<string> output)
{
progressMonitor.RunningProcess($"{Exec} {args}");
var pi = MakeDotnetStartInfo(args);
var threadId = $"[{Environment.CurrentManagedThreadId:D3}]";
void onOut(string s)
{
Console.Out.WriteLine($"{threadId} {s}");
}
void onError(string s)
{
Console.Error.WriteLine($"{threadId} {s}");
}
var exitCode = pi.ReadOutput(out output, onOut, onError);
if (exitCode != 0)
{
progressMonitor.CommandFailed(Exec, args, exitCode);
return false;
}
return true;
}
public bool RunCommand(string args) =>
RunCommandAux(args, out _);
public bool RunCommand(string args, out IList<string> output) =>
RunCommandAux(args, out output);
}
}

6
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotnetVersion.cs → csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetVersion.cs
View File

@@ -3,7 +3,7 @@ using System.IO;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal record DotnetVersion : IComparable<DotnetVersion>
internal record DotNetVersion : IComparable<DotNetVersion>
{
private readonly string dir;
private readonly Version version;
@@ -48,7 +48,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
public DotnetVersion(string dir, string version, string preReleaseVersionType, string preReleaseVersion)
public DotNetVersion(string dir, string version, string preReleaseVersionType, string preReleaseVersion)
{
this.dir = dir;
this.version = Version.Parse(version);
@@ -59,7 +59,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
public int CompareTo(DotnetVersion? other)
public int CompareTo(DotNetVersion? other)
{
var c = version.CompareTo(other?.version);
if (c == 0 && IsPreRelease)

56
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs
View File

@@ -19,6 +19,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private readonly IUnsafeFileReader unsafeFileReader;
private readonly IEnumerable<string> files;
private readonly HashSet<string> allPackages = new HashSet<string>();
private readonly HashSet<string> implicitUsingNamespaces = new HashSet<string>();
private readonly Initializer initialize;
public HashSet<string> AllPackages
@@ -48,6 +49,26 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
private bool useImplicitUsings = false;
public bool UseImplicitUsings
{
get
{
initialize.Run();
return useImplicitUsings;
}
}
public HashSet<string> CustomImplicitUsings
{
get
{
initialize.Run();
return implicitUsingNamespaces;
}
}
internal FileContent(ProgressMonitor progressMonitor,
IEnumerable<string> files,
IUnsafeFileReader unsafeFileReader)
@@ -62,7 +83,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public FileContent(ProgressMonitor progressMonitor, IEnumerable<string> files) : this(progressMonitor, files, new UnsafeFileReader())
{ }
private static string GetGroup(ReadOnlySpan<char> input, ValueMatch valueMatch, string groupPrefix)
private static string GetGroup(ReadOnlySpan<char> input, ValueMatch valueMatch, string groupPrefix, bool toLower)
{
var match = input.Slice(valueMatch.Index, valueMatch.Length);
var includeIndex = match.IndexOf(groupPrefix, StringComparison.InvariantCultureIgnoreCase);
@@ -76,7 +97,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var quoteIndex1 = match.IndexOf("\"");
var quoteIndex2 = match.Slice(quoteIndex1 + 1).IndexOf("\"");
return match.Slice(quoteIndex1 + 1, quoteIndex2).ToString().ToLowerInvariant();
var result = match.Slice(quoteIndex1 + 1, quoteIndex2).ToString();
if (toLower)
{
result = result.ToLowerInvariant();
}
return result;
}
private static bool IsGroupMatch(ReadOnlySpan<char> line, Regex regex, string groupPrefix, string value)
@@ -84,7 +112,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
foreach (var valueMatch in regex.EnumerateMatches(line))
{
// We can't get the group from the ValueMatch, so doing it manually:
if (GetGroup(line, valueMatch, groupPrefix) == value.ToLowerInvariant())
if (GetGroup(line, valueMatch, groupPrefix, toLower: true) == value.ToLowerInvariant())
{
return true;
}
@@ -105,7 +133,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
foreach (var valueMatch in PackageReference().EnumerateMatches(line))
{
// We can't get the group from the ValueMatch, so doing it manually:
var packageName = GetGroup(line, valueMatch, "Include");
var packageName = GetGroup(line, valueMatch, "Include", toLower: true);
if (!string.IsNullOrEmpty(packageName))
{
allPackages.Add(packageName);
@@ -119,6 +147,23 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
IsGroupMatch(line, ProjectSdk(), "Sdk", "Microsoft.NET.Sdk.Web") ||
IsGroupMatch(line, FrameworkReference(), "Include", "Microsoft.AspNetCore.App");
}
// Determine if implicit usings are used.
if (!useImplicitUsings)
{
useImplicitUsings = line.Contains("<ImplicitUsings>enable</ImplicitUsings>".AsSpan(), StringComparison.Ordinal) ||
line.Contains("<ImplicitUsings>true</ImplicitUsings>".AsSpan(), StringComparison.Ordinal);
}
// Find all custom implicit usings.
foreach (var valueMatch in CustomImplicitUsingDeclarations().EnumerateMatches(line))
{
var ns = GetGroup(line, valueMatch, "Include", toLower: false);
if (!string.IsNullOrEmpty(ns))
{
implicitUsingNamespaces.Add(ns);
}
}
}
}
catch (Exception ex)
@@ -136,6 +181,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
[GeneratedRegex("<(.*\\s)?Project.*\\sSdk=\"(.*?)\".*/?>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
private static partial Regex ProjectSdk();
[GeneratedRegex("<Using.*\\sInclude=\"(.*?)\".*/?>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
private static partial Regex CustomImplicitUsingDeclarations();
}
internal interface IUnsafeFileReader

61
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Git.cs
View File

@@ -1,61 +0,0 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
/// <summary>
/// Utilities for querying information from the git CLI.
/// </summary>
internal class Git
{
private readonly ProgressMonitor progressMonitor;
private const string git = "git";
public Git(ProgressMonitor progressMonitor)
{
this.progressMonitor = progressMonitor;
}
/// <summary>
/// Lists all files matching <paramref name="pattern"/> which are tracked in the
/// current git repository.
/// </summary>
/// <param name="pattern">The file pattern.</param>
/// <returns>A list of all tracked files which match <paramref name="pattern"/>.</returns>
/// <exception cref="Exception"></exception>
public List<string> ListFiles(string pattern)
{
var results = new List<string>();
var args = string.Join(' ', "ls-files", $"\"{pattern}\"");
progressMonitor.RunningProcess($"{git} {args}");
var pi = new ProcessStartInfo(git, args)
{
UseShellExecute = false,
RedirectStandardOutput = true
};
using var p = new Process() { StartInfo = pi };
p.OutputDataReceived += new DataReceivedEventHandler((sender, e) =>
{
if (!string.IsNullOrWhiteSpace(e.Data))
{
results.Add(e.Data);
}
});
p.Start();
p.BeginOutputReadLine();
p.WaitForExit();
if (p.ExitCode != 0)
{
progressMonitor.CommandFailed(git, args, p.ExitCode);
throw new Exception($"{git} {args} failed");
}
return results;
}
}
}

2
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs
View File

@@ -4,7 +4,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal interface IDotNet
{
bool RestoreProjectToDirectory(string project, string directory, out string stdout, string? pathToNugetConfig = null);
bool RestoreProjectToDirectory(string project, string directory, string? pathToNugetConfig = null);
bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, out IEnumerable<string> projects);
bool New(string folder);
bool AddPackage(string folder, string package);

23
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNetCliInvoker.cs Normal file
View File

@@ -0,0 +1,23 @@
using System.Collections.Generic;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal interface IDotNetCliInvoker
{
/// <summary>
/// The name of the dotnet executable.
/// </summary>
string Exec { get; }
/// <summary>
/// Execute `dotnet <args>` and return true if the command succeeded, otherwise false.
/// </summary>
bool RunCommand(string args);
/// <summary>
/// Execute `dotnet <args>` and return true if the command succeeded, otherwise false.
/// The output of the command is returned in `output`.
/// </summary>
bool RunCommand(string args, out IList<string> output);
}
}

3
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs
View File

@@ -57,6 +57,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public void ResolvedReference(string filename) =>
LogInfo($"Resolved {filename}");
public void RemovedReference(string filename) =>
LogInfo($"Reference {filename} has been removed");
public void Summary(int existingSources, int usedSources, int missingSources,
int references, int unresolvedReferences,
int resolvedConflicts, int totalProjects, int failedProjects,

6
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs
View File

@@ -8,13 +8,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal class Razor
{
private readonly DotnetVersion sdk;
private readonly DotNetVersion sdk;
private readonly ProgressMonitor progressMonitor;
private readonly DotNet dotNet;
private readonly IDotNet dotNet;
private readonly string sourceGeneratorFolder;
private readonly string cscPath;
public Razor(DotnetVersion sdk, DotNet dotNet, ProgressMonitor progressMonitor)
public Razor(DotNetVersion sdk, IDotNet dotNet, ProgressMonitor progressMonitor)
{
this.sdk = sdk;
this.progressMonitor = progressMonitor;

12
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Runtime.cs
View File

@@ -17,8 +17,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private const string aspNetCoreApp = "Microsoft.AspNetCore.App";
private readonly IDotNet dotNet;
private readonly Lazy<Dictionary<string, DotnetVersion>> newestRuntimes;
private Dictionary<string, DotnetVersion> NewestRuntimes => newestRuntimes.Value;
private readonly Lazy<Dictionary<string, DotNetVersion>> newestRuntimes;
private Dictionary<string, DotNetVersion> NewestRuntimes => newestRuntimes.Value;
private static string ExecutingRuntime => RuntimeEnvironment.GetRuntimeDirectory();
public Runtime(IDotNet dotNet)
@@ -36,17 +36,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// It is assume that the format of a listed runtime is something like:
/// Microsoft.NETCore.App 7.0.2 [/usr/share/dotnet/shared/Microsoft.NETCore.App]
/// </summary>
private static Dictionary<string, DotnetVersion> ParseRuntimes(IList<string> listed)
private static Dictionary<string, DotNetVersion> ParseRuntimes(IList<string> listed)
{
// Parse listed runtimes.
var runtimes = new Dictionary<string, DotnetVersion>();
var runtimes = new Dictionary<string, DotNetVersion>();
var regex = RuntimeRegex();
listed.ForEach(r =>
{
var match = regex.Match(r);
if (match.Success)
{
runtimes.AddOrUpdateToLatest(match.Groups[1].Value, new DotnetVersion(match.Groups[6].Value, match.Groups[2].Value, match.Groups[4].Value, match.Groups[5].Value));
runtimes.AddOrUpdateToLatest(match.Groups[1].Value, new DotNetVersion(match.Groups[6].Value, match.Groups[2].Value, match.Groups[4].Value, match.Groups[5].Value));
}
});
@@ -56,7 +56,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// <summary>
/// Returns a dictionary mapping runtimes to their newest version.
/// </summary>
internal Dictionary<string, DotnetVersion> GetNewestRuntimes()
internal Dictionary<string, DotNetVersion> GetNewestRuntimes()
{
var listed = dotNet.GetListedRuntimes();
return ParseRuntimes(listed);

8
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Sdk.cs
View File

@@ -14,23 +14,23 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
[GeneratedRegex(@"^(\d+\.\d+\.\d+)(-([a-z]+)\.(\d+\.\d+\.\d+))?\s\[(.+)\]$")]
private static partial Regex SdkRegex();
private static HashSet<DotnetVersion> ParseSdks(IList<string> listed)
private static HashSet<DotNetVersion> ParseSdks(IList<string> listed)
{
var sdks = new HashSet<DotnetVersion>();
var sdks = new HashSet<DotNetVersion>();
var regex = SdkRegex();
listed.ForEach(r =>
{
var match = regex.Match(r);
if (match.Success)
{
sdks.Add(new DotnetVersion(match.Groups[5].Value, match.Groups[1].Value, match.Groups[3].Value, match.Groups[4].Value));
sdks.Add(new DotNetVersion(match.Groups[5].Value, match.Groups[1].Value, match.Groups[3].Value, match.Groups[4].Value));
}
});
return sdks;
}
public DotnetVersion? GetNewestSdk()
public DotNetVersion? GetNewestSdk()
{
var listed = dotNet.GetListedSdks();
var sdks = ParseSdks(listed);

9
csharp/extractor/Semmle.Extraction.CSharp.DependencyStubGenerator/Program.cs Normal file
View File

@@ -0,0 +1,9 @@
using Semmle.Extraction.CSharp.DependencyFetching;
using Semmle.Extraction.CSharp.StubGenerator;
using Semmle.Util.Logging;
var logger = new ConsoleLogger(Verbosity.Info, logThreadId: false);
using var dependencyManager = new DependencyManager(".", DependencyOptions.Default, logger);
StubGenerator.GenerateStubs(logger, dependencyManager.ReferenceFiles, "codeql_csharp_stubs");
return 0;

18
csharp/extractor/Semmle.Extraction.CSharp.DependencyStubGenerator/Semmle.Extraction.CSharp.DependencyStubGenerator.csproj Normal file
View File

@@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net7.0</TargetFramework>
<AssemblyName>Semmle.Extraction.CSharp.DependencyStubGenerator</AssemblyName>
<RootNamespace>Semmle.Extraction.CSharp.DependencyStubGenerator</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
<ProjectReference Include="..\Semmle.Extraction.CSharp.DependencyFetching\Semmle.Extraction.CSharp.DependencyFetching.csproj" />
<ProjectReference Include="..\Semmle.Extraction.CSharp.StubGenerator\Semmle.Extraction.CSharp.StubGenerator.csproj" />
</ItemGroup>
</Project>

2
csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs
View File

@@ -119,7 +119,7 @@ namespace Semmle.Extraction.CSharp.Standalone
var stopwatch = new Stopwatch();
stopwatch.Start();
using var logger = new ConsoleLogger(options.Verbosity);
using var logger = new ConsoleLogger(options.Verbosity, logThreadId: true);
logger.Log(Severity.Info, "Running C# standalone extractor");
using var a = new Analysis(logger, options);
var sourceFileCount = a.Extraction.Sources.Count;

5
csharp/extractor/Semmle.Extraction.CSharp.Standalone/Options.cs
View File

@@ -2,6 +2,7 @@ using System.IO;
using Semmle.Util;
using Semmle.Util.Logging;
using Semmle.Extraction.CSharp.DependencyFetching;
using System;
namespace Semmle.Extraction.CSharp.Standalone
{
@@ -64,7 +65,7 @@ namespace Semmle.Extraction.CSharp.Standalone
var fi = new FileInfo(dependencies.SolutionFile);
if (!fi.Exists)
{
System.Console.WriteLine("Error: The solution {0} does not exist", fi.FullName);
System.Console.WriteLine($"[{Environment.CurrentManagedThreadId:D3}] Error: The solution {fi.FullName} does not exist");
Errors = true;
}
return true;
@@ -72,7 +73,7 @@ namespace Semmle.Extraction.CSharp.Standalone
public override void InvalidArgument(string argument)
{
System.Console.WriteLine($"Error: Invalid argument {argument}");
System.Console.WriteLine($"[{Environment.CurrentManagedThreadId:D3}] Error: Invalid argument {argument}");
Errors = true;
}

33
csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/RelevantSymbol.cs Normal file
View File

@@ -0,0 +1,33 @@
using System.Linq;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
using Semmle.Util;
namespace Semmle.Extraction.CSharp.StubGenerator;
internal class RelevantSymbol
{
private readonly IAssemblySymbol assembly;
private readonly MemoizedFunc<INamedTypeSymbol, bool> isRelevantNamedType;
private readonly MemoizedFunc<INamespaceSymbol, bool> isRelevantNamespace;
public RelevantSymbol(IAssemblySymbol assembly)
{
this.assembly = assembly;
isRelevantNamedType = new(symbol =>
StubVisitor.IsRelevantBaseType(symbol) &&
SymbolEqualityComparer.Default.Equals(symbol.ContainingAssembly, assembly));
isRelevantNamespace = new(symbol =>
symbol.GetTypeMembers().Any(IsRelevantNamedType) ||
symbol.GetNamespaceMembers().Any(IsRelevantNamespace));
}
public bool IsRelevantNamedType(INamedTypeSymbol symbol) => isRelevantNamedType.Invoke(symbol);
public bool IsRelevantNamespace(INamespaceSymbol symbol) => isRelevantNamespace.Invoke(symbol);
}

18
csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/Semmle.Extraction.CSharp.StubGenerator.csproj Normal file
View File

@@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<AssemblyName>Semmle.Extraction.CSharp.StubGenerator</AssemblyName>
<RootNamespace>Semmle.Extraction.CSharp.StubGenerator</RootNamespace>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
<ProjectReference Include="..\Semmle.Extraction.CSharp.DependencyFetching\Semmle.Extraction.CSharp.DependencyFetching.csproj" />
<ProjectReference Include="..\Semmle.Extraction.CSharp.Util\Semmle.Extraction.CSharp.Util.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.4.0" />
</ItemGroup>
</Project>

86
csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/StubGenerator.cs Normal file
View File

@@ -0,0 +1,86 @@
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.StubGenerator;
public static class StubGenerator
{
/// <summary>
/// Generates stubs for all the provided assembly paths.
/// </summary>
/// <param name="referencesPaths">The paths of the assemblies to generate stubs for.</param>
/// <param name="outputPath">The path in which to store the stubs.</param>
public static string[] GenerateStubs(ILogger logger, IEnumerable<string> referencesPaths, string outputPath)
{
var stopWatch = new System.Diagnostics.Stopwatch();
stopWatch.Start();
var threads = EnvironmentVariables.GetDefaultNumberOfThreads();
using var stubPaths = new BlockingCollection<string>();
using var references = new BlockingCollection<(MetadataReference Reference, string Path)>();
Parallel.ForEach(referencesPaths, new ParallelOptions { MaxDegreeOfParallelism = threads }, path =>
{
var reference = MetadataReference.CreateFromFile(path);
references.Add((reference, path));
});
logger.Log(Severity.Info, $"Generating stubs for {references.Count} assemblies.");
var compilation = CSharpCompilation.Create(
"stubgenerator.dll",
null,
references.Select(tuple => tuple.Item1),
new CSharpCompilationOptions(OutputKind.ConsoleApplication, allowUnsafe: true));
Parallel.ForEach(references, new ParallelOptions { MaxDegreeOfParallelism = threads }, @ref =>
{
StubReference(logger, compilation, outputPath, @ref.Reference, @ref.Path, stubPaths);
});
stopWatch.Stop();
logger.Log(Severity.Info, $"Stub generation took {stopWatch.Elapsed}.");
return stubPaths.ToArray();
}
private static void StubReference(ILogger logger, CSharpCompilation compilation, string outputPath, MetadataReference reference, string path, BlockingCollection<string> stubPaths)
{
if (compilation.GetAssemblyOrModuleSymbol(reference) is not IAssemblySymbol assembly)
return;
var relevantSymbol = new RelevantSymbol(assembly);
if (!assembly.Modules.Any(m => relevantSymbol.IsRelevantNamespace(m.GlobalNamespace)))
return;
var stubPath = FileUtils.NestPaths(logger, outputPath, path.Replace(".dll", ".cs"));
stubPaths.Add(stubPath);
using var fileStream = new FileStream(stubPath, FileMode.Create, FileAccess.Write);
using var writer = new StreamWriter(fileStream, new UTF8Encoding(false));
var visitor = new StubVisitor(writer, relevantSymbol);
writer.WriteLine("// This file contains auto-generated code.");
writer.WriteLine($"// Generated from `{assembly.Identity}`.");
visitor.StubAttributes(assembly.GetAttributes(), "assembly: ");
foreach (var module in assembly.Modules)
{
module.GlobalNamespace.Accept(visitor);
}
}
}

829
csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/StubVisitor.cs Normal file
View File

@@ -0,0 +1,829 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Microsoft.CodeAnalysis;
using Semmle.Extraction.CSharp.Util;
namespace Semmle.Extraction.CSharp.StubGenerator;
internal sealed class StubVisitor : SymbolVisitor
{
private readonly TextWriter stubWriter;
private readonly RelevantSymbol relevantSymbol;
public StubVisitor(TextWriter stubWriter, RelevantSymbol relevantSymbol)
{
this.stubWriter = stubWriter;
this.relevantSymbol = relevantSymbol;
}
private static bool IsNotPublic(Accessibility accessibility) =>
accessibility == Accessibility.Private ||
accessibility == Accessibility.Internal ||
accessibility == Accessibility.ProtectedAndInternal;
public static bool IsRelevantBaseType(INamedTypeSymbol symbol) =>
!IsNotPublic(symbol.DeclaredAccessibility) &&
symbol.CanBeReferencedByName;
private void StubExplicitInterface(ISymbol symbol, ISymbol? explicitInterfaceSymbol, bool writeName = true)
{
static bool ContainsTupleType(ITypeSymbol type) =>
type is INamedTypeSymbol named && (named.IsTupleType || named.TypeArguments.Any(ContainsTupleType)) ||
type is IArrayTypeSymbol array && ContainsTupleType(array.ElementType) ||
type is IPointerTypeSymbol pointer && ContainsTupleType(pointer.PointedAtType);
static bool EqualsModuloTupleElementNames(ITypeSymbol t1, ITypeSymbol t2) =>
SymbolEqualityComparer.Default.Equals(t1, t2) ||
(
t1 is INamedTypeSymbol named1 &&
t2 is INamedTypeSymbol named2 &&
EqualsModuloTupleElementNames(named1.ConstructedFrom, named2.ConstructedFrom) &&
named1.TypeArguments.Length == named2.TypeArguments.Length &&
named1.TypeArguments.Zip(named2.TypeArguments).All(p => EqualsModuloTupleElementNames(p.First, p.Second))
) ||
(
t1 is IArrayTypeSymbol array1 &&
t2 is IArrayTypeSymbol array2 &&
EqualsModuloTupleElementNames(array1.ElementType, array2.ElementType)
) ||
(
t1 is IPointerTypeSymbol pointer1 &&
t2 is IPointerTypeSymbol pointer2 &&
EqualsModuloTupleElementNames(pointer1.PointedAtType, pointer2.PointedAtType)
);
if (explicitInterfaceSymbol is not null)
{
var explicitInterfaceType = explicitInterfaceSymbol.ContainingType;
// Workaround for when the explicit interface type contains named tuple types,
// in which case Roslyn may incorrectly forget the names of the tuple elements.
//
// For example, without this workaround we would incorrectly generate the following stub:
//
// ```csharp
// public sealed class UnorderedItemsCollection : System.Collections.Generic.IEnumerable<(TElement Element, TPriority Priority)>, ...
// {
// System.Collections.Generic.IEnumerator<(TElement Element, TPriority Priority)> System.Collections.Generic.IEnumerable<(TElement, TPriority)>.GetEnumerator() => throw null;
// }
// ```
if (ContainsTupleType(explicitInterfaceType))
{
explicitInterfaceType = symbol.ContainingType.Interfaces.First(i => ContainsTupleType(i) && EqualsModuloTupleElementNames(i, explicitInterfaceSymbol.ContainingType));
}
stubWriter.Write(explicitInterfaceType.GetQualifiedName());
stubWriter.Write('.');
if (writeName)
stubWriter.Write(explicitInterfaceSymbol.GetName());
}
else if (writeName)
{
stubWriter.Write(symbol.GetName());
}
}
private void StubAccessibility(Accessibility accessibility)
{
switch (accessibility)
{
case Accessibility.Public:
stubWriter.Write("public ");
break;
case Accessibility.Protected or Accessibility.ProtectedOrInternal:
stubWriter.Write("protected ");
break;
case Accessibility.Internal:
stubWriter.Write("internal ");
break;
case Accessibility.ProtectedAndInternal:
stubWriter.Write("protected internal ");
break;
default:
stubWriter.Write($"/* TODO: {accessibility} */");
break;
}
}
private void StubModifiers(ISymbol symbol, bool skipAccessibility = false)
{
if (symbol.ContainingType is ITypeSymbol containing && containing.TypeKind == TypeKind.Interface)
skipAccessibility = true;
if (symbol is IMethodSymbol method && method.MethodKind == MethodKind.Constructor && symbol.IsStatic)
skipAccessibility = true;
if (!skipAccessibility)
StubAccessibility(symbol.DeclaredAccessibility);
if (symbol.IsAbstract)
{
if (
// exclude interface declarations
(symbol is not INamedTypeSymbol type || type.TypeKind != TypeKind.Interface) &&
// exclude non-static interface members
(symbol.ContainingType is not INamedTypeSymbol containingType || containingType.TypeKind != TypeKind.Interface || symbol.IsStatic))
{
stubWriter.Write("abstract ");
}
}
if (symbol.IsStatic && !(symbol is IFieldSymbol field && field.IsConst))
stubWriter.Write("static ");
if (symbol.IsVirtual)
stubWriter.Write("virtual ");
if (symbol.IsOverride)
stubWriter.Write("override ");
if (symbol.IsSealed)
{
if (!(symbol is INamedTypeSymbol type && (type.TypeKind == TypeKind.Enum || type.TypeKind == TypeKind.Delegate || type.TypeKind == TypeKind.Struct)))
stubWriter.Write("sealed ");
}
if (symbol.IsExtern)
stubWriter.Write("extern ");
}
private void StubTypedConstant(TypedConstant c)
{
switch (c.Kind)
{
case TypedConstantKind.Primitive:
if (c.Value is string s)
{
stubWriter.Write($"\"{s}\"");
}
else if (c.Value is char ch)
{
stubWriter.Write($"'{ch}'");
}
else if (c.Value is bool b)
{
stubWriter.Write(b ? "true" : "false");
}
else if (c.Value is int i)
{
stubWriter.Write(i);
}
else if (c.Value is long l)
{
stubWriter.Write(l);
}
else if (c.Value is float f)
{
stubWriter.Write(f);
}
else if (c.Value is double d)
{
stubWriter.Write(d);
}
else
{
stubWriter.Write("throw null");
}
break;
case TypedConstantKind.Enum:
stubWriter.Write("throw null");
break;
case TypedConstantKind.Array:
stubWriter.Write("new []{");
WriteCommaSep(c.Values, StubTypedConstant);
stubWriter.Write("}");
break;
default:
stubWriter.Write($"/* TODO: {c.Kind} */ throw null");
break;
}
}
private static readonly HashSet<string> attributeAllowList = new() {
"System.FlagsAttribute"
};
private void StubAttribute(AttributeData a, string prefix)
{
if (a.AttributeClass is not INamedTypeSymbol @class)
return;
var qualifiedName = @class.GetQualifiedName();
if (!attributeAllowList.Contains(qualifiedName))
return;
if (qualifiedName.EndsWith("Attribute"))
qualifiedName = qualifiedName[..^9];
stubWriter.Write($"[{prefix}{qualifiedName}");
if (a.ConstructorArguments.Any())
{
stubWriter.Write("(");
WriteCommaSep(a.ConstructorArguments, StubTypedConstant);
stubWriter.Write(")");
}
stubWriter.WriteLine("]");
}
public void StubAttributes(IEnumerable<AttributeData> a, string prefix = "")
{
foreach (var attribute in a)
{
StubAttribute(attribute, prefix);
}
}
private void StubEvent(IEventSymbol symbol, IEventSymbol? explicitInterfaceSymbol)
{
StubAttributes(symbol.GetAttributes());
StubModifiers(symbol, explicitInterfaceSymbol is not null);
stubWriter.Write("event ");
stubWriter.Write(symbol.Type.GetQualifiedName());
stubWriter.Write(" ");
StubExplicitInterface(symbol, explicitInterfaceSymbol);
if (explicitInterfaceSymbol is null)
{
stubWriter.WriteLine(";");
}
else
{
stubWriter.Write(" { ");
stubWriter.Write("add {} ");
stubWriter.Write("remove {} ");
stubWriter.WriteLine("}");
}
}
private static T[] FilterExplicitInterfaceImplementations<T>(IEnumerable<T> explicitInterfaceImplementations) where T : ISymbol =>
explicitInterfaceImplementations.Where(i => IsRelevantBaseType(i.ContainingType)).ToArray();
public override void VisitEvent(IEventSymbol symbol)
{
var explicitInterfaceImplementations = FilterExplicitInterfaceImplementations(symbol.ExplicitInterfaceImplementations);
if (IsNotPublic(symbol.DeclaredAccessibility) && explicitInterfaceImplementations.Length == 0)
return;
foreach (var explicitInterfaceSymbol in explicitInterfaceImplementations)
{
StubEvent(symbol, explicitInterfaceSymbol);
}
if (explicitInterfaceImplementations.Length == 0)
StubEvent(symbol, null);
}
private static bool IsUnsafe(ITypeSymbol symbol) =>
symbol.TypeKind == TypeKind.Pointer ||
symbol.TypeKind == TypeKind.FunctionPointer ||
(symbol is INamedTypeSymbol named && named.TypeArguments.Any(IsUnsafe)) ||
(symbol is IArrayTypeSymbol at && IsUnsafe(at.ElementType));
private static readonly HashSet<string> keywords = new() {
"abstract", "as", "base", "bool", "break", "byte", "case", "catch", "char", "checked",
"class", "const", "continue", "decimal", "default", "delegate", "do", "double", "else",
"enum", "event", "explicit", "extern", "false", "finally", "fixed", "float", "for", "foreach",
"goto", "if", "implicit", "in", "int", "interface", "internal", "is", "lock", "long",
"namespace", "new", "null", "object", "operator", "out", "override", "params", "private",
"protected", "public", "readonly", "ref", "return", "sbyte", "sealed", "short", "sizeof",
"stackalloc", "static", "string", "struct", "switch", "this", "throw", "true", "try",
"typeof", "uint", "ulong", "unchecked", "unsafe", "ushort", "using", "virtual", "void",
"volatile", "while"
};
private static string EscapeIdentifier(string identifier) =>
keywords.Contains(identifier) ? "@" + identifier : identifier;
public override void VisitField(IFieldSymbol symbol)
{
if (IsNotPublic(symbol.DeclaredAccessibility))
return;
StubAttributes(symbol.GetAttributes());
StubModifiers(symbol);
if (symbol.IsConst)
stubWriter.Write("const ");
if (IsUnsafe(symbol.Type))
{
stubWriter.Write("unsafe ");
}
stubWriter.Write(symbol.Type.GetQualifiedName());
stubWriter.Write(" ");
stubWriter.Write(EscapeIdentifier(symbol.Name));
if (symbol.IsConst)
stubWriter.Write(" = default");
stubWriter.WriteLine(";");
}
private void WriteCommaSep<T>(IEnumerable<T> items, Action<T> writeItem)
{
var first = true;
foreach (var item in items)
{
if (!first)
{
stubWriter.Write(", ");
}
writeItem(item);
first = false;
}
}
private void WriteStringCommaSep<T>(IEnumerable<T> items, Func<T, string> writeItem)
{
WriteCommaSep(items, item => stubWriter.Write(writeItem(item)));
}
private void StubTypeParameters(IEnumerable<ITypeParameterSymbol> typeParameters)
{
if (!typeParameters.Any())
return;
stubWriter.Write('<');
WriteStringCommaSep(typeParameters, typeParameter => typeParameter.Name);
stubWriter.Write('>');
}
private void StubTypeParameterConstraints(IEnumerable<ITypeParameterSymbol> typeParameters)
{
if (!typeParameters.Any())
return;
var inheritsConstraints = typeParameters.Any(tp =>
tp.DeclaringMethod is IMethodSymbol method &&
(method.IsOverride || method.ExplicitInterfaceImplementations.Any()));
foreach (var typeParameter in typeParameters)
{
var firstTypeParameterConstraint = true;
void WriteTypeParameterConstraint(Action a)
{
if (firstTypeParameterConstraint)
{
stubWriter.Write($" where {typeParameter.Name} : ");
}
else
{
stubWriter.Write(", ");
}
a();
firstTypeParameterConstraint = false;
}
if (typeParameter.HasReferenceTypeConstraint)
{
WriteTypeParameterConstraint(() => stubWriter.Write("class"));
}
if (typeParameter.HasValueTypeConstraint &&
!typeParameter.HasUnmanagedTypeConstraint &&
!typeParameter.ConstraintTypes.Any(t => t.GetQualifiedName() is "System.Enum"))
{
WriteTypeParameterConstraint(() => stubWriter.Write("struct"));
}
if (inheritsConstraints)
continue;
if (typeParameter.HasUnmanagedTypeConstraint)
{
WriteTypeParameterConstraint(() => stubWriter.Write("unmanaged"));
}
var constraintTypes = typeParameter.ConstraintTypes.Select(t => t.GetQualifiedName()).Where(s => s is not "").ToArray();
if (constraintTypes.Any())
{
WriteTypeParameterConstraint(() =>
{
WriteStringCommaSep(constraintTypes, constraintType => constraintType);
});
}
if (typeParameter.HasConstructorConstraint)
{
WriteTypeParameterConstraint(() => stubWriter.Write("new()"));
}
}
}
private static INamedTypeSymbol? GetBaseType(INamedTypeSymbol symbol)
{
if (symbol.BaseType is INamedTypeSymbol @base &&
@base.SpecialType != SpecialType.System_Object &&
@base.SpecialType != SpecialType.System_ValueType)
{
return @base;
}
return null;
}
private static IMethodSymbol? GetBaseConstructor(INamedTypeSymbol symbol)
{
if (GetBaseType(symbol) is not INamedTypeSymbol @base)
return null;
var containingTypes = new HashSet<INamedTypeSymbol>(SymbolEqualityComparer.Default);
var current = symbol;
while (current is not null)
{
containingTypes.Add(current);
current = current.ContainingType;
}
var baseCtor = @base.Constructors.
Where(c => !c.IsStatic).
Where(c =>
c.DeclaredAccessibility == Accessibility.Public ||
c.DeclaredAccessibility == Accessibility.Protected ||
c.DeclaredAccessibility == Accessibility.ProtectedOrInternal ||
containingTypes.Contains(c.ContainingType)
).
MinBy(c => c.Parameters.Length);
return baseCtor?.Parameters.Length > 0 ? baseCtor : null;
}
private static IMethodSymbol? GetBaseConstructor(IMethodSymbol ctor)
{
if (ctor.MethodKind != MethodKind.Constructor)
return null;
return GetBaseConstructor(ctor.ContainingType);
}
private void StubParameters(ICollection<IParameterSymbol> parameters)
{
WriteCommaSep(parameters, parameter =>
{
switch (parameter.RefKind)
{
case RefKind.None:
break;
case RefKind.Ref:
stubWriter.Write("ref ");
break;
case RefKind.Out:
stubWriter.Write("out ");
break;
case RefKind.In:
stubWriter.Write("in ");
break;
default:
stubWriter.Write($"/* TODO: {parameter.RefKind} */");
break;
}
if (parameter.IsParams)
stubWriter.Write("params ");
stubWriter.Write(parameter.Type.GetQualifiedName());
stubWriter.Write(" ");
stubWriter.Write(EscapeIdentifier(parameter.Name));
if (parameter.HasExplicitDefaultValue)
{
stubWriter.Write(" = ");
stubWriter.Write($"default({parameter.Type.GetQualifiedName()})");
}
});
}
private void StubMethod(IMethodSymbol symbol, IMethodSymbol? explicitInterfaceSymbol, IMethodSymbol? baseCtor)
{
var methodKind = explicitInterfaceSymbol is null ? symbol.MethodKind : explicitInterfaceSymbol.MethodKind;
var relevantMethods = new[] {
MethodKind.Constructor,
MethodKind.Conversion,
MethodKind.UserDefinedOperator,
MethodKind.Ordinary
};
if (!relevantMethods.Contains(methodKind))
return;
StubAttributes(symbol.GetAttributes());
StubModifiers(symbol, explicitInterfaceSymbol is not null);
if (IsUnsafe(symbol.ReturnType) || symbol.Parameters.Any(p => IsUnsafe(p.Type)))
{
stubWriter.Write("unsafe ");
}
if (methodKind == MethodKind.Constructor)
{
stubWriter.Write(symbol.ContainingType.Name);
}
else if (methodKind == MethodKind.Conversion)
{
if (!symbol.TryGetOperatorSymbol(out var operatorName))
{
stubWriter.WriteLine($"/* TODO: {symbol.Name} */");
return;
}
switch (operatorName)
{
case "explicit conversion":
stubWriter.Write("explicit operator ");
break;
case "checked explicit conversion":
stubWriter.Write("explicit operator checked ");
break;
case "implicit conversion":
stubWriter.Write("implicit operator ");
break;
case "checked implicit conversion":
stubWriter.Write("implicit operator checked ");
break;
default:
stubWriter.Write($"/* TODO: {symbol.Name} */");
break;
}
stubWriter.Write(symbol.ReturnType.GetQualifiedName());
}
else if (methodKind == MethodKind.UserDefinedOperator)
{
if (!symbol.TryGetOperatorSymbol(out var operatorName))
{
stubWriter.WriteLine($"/* TODO: {symbol.Name} */");
return;
}
stubWriter.Write(symbol.ReturnType.GetQualifiedName());
stubWriter.Write(" ");
StubExplicitInterface(symbol, explicitInterfaceSymbol, writeName: false);
stubWriter.Write("operator ");
stubWriter.Write(operatorName);
}
else
{
stubWriter.Write(symbol.ReturnType.GetQualifiedName());
stubWriter.Write(" ");
StubExplicitInterface(symbol, explicitInterfaceSymbol);
StubTypeParameters(symbol.TypeParameters);
}
stubWriter.Write("(");
if (symbol.IsExtensionMethod)
{
stubWriter.Write("this ");
}
StubParameters(symbol.Parameters);
stubWriter.Write(")");
if (baseCtor is not null)
{
stubWriter.Write(" : base(");
WriteStringCommaSep(baseCtor.Parameters, parameter => $"default({parameter.Type.GetQualifiedName()})");
stubWriter.Write(")");
}
StubTypeParameterConstraints(symbol.TypeParameters);
if (symbol.IsAbstract)
stubWriter.WriteLine(";");
else
stubWriter.WriteLine(" => throw null;");
}
public override void VisitMethod(IMethodSymbol symbol)
{
var baseCtor = GetBaseConstructor(symbol);
var explicitInterfaceImplementations = FilterExplicitInterfaceImplementations(symbol.ExplicitInterfaceImplementations);
if (baseCtor is null &&
((IsNotPublic(symbol.DeclaredAccessibility) && explicitInterfaceImplementations.Length == 0) ||
symbol.IsImplicitlyDeclared))
{
return;
}
foreach (var explicitInterfaceSymbol in explicitInterfaceImplementations)
{
StubMethod(symbol, explicitInterfaceSymbol, baseCtor);
}
// Roslyn reports certain methods to be only explicit interface methods, such as
// `System.Numerics.INumberBase<int>.TryParse(string s, System.Globalization.NumberStyles style, System.IFormatProvider provider, out int result)`
// in the `System.Int32` struct. However, we also need a non-explicit implementation
// in order for things to compile.
var roslynExplicitInterfaceWorkaround =
symbol.ContainingType.GetQualifiedName() is "int" &&
explicitInterfaceImplementations.Any(i => i.ContainingType.GetQualifiedName() is "System.Numerics.INumberBase<int>");
if (explicitInterfaceImplementations.Length == 0 || roslynExplicitInterfaceWorkaround)
StubMethod(symbol, null, baseCtor);
}
public override void VisitNamedType(INamedTypeSymbol symbol)
{
if (!relevantSymbol.IsRelevantNamedType(symbol))
{
return;
}
if (symbol.TypeKind == TypeKind.Delegate)
{
var invokeMethod = symbol.DelegateInvokeMethod!;
StubAttributes(symbol.GetAttributes());
StubModifiers(symbol);
if (IsUnsafe(invokeMethod.ReturnType) || invokeMethod.Parameters.Any(p => IsUnsafe(p.Type)))
{
stubWriter.Write("unsafe ");
}
stubWriter.Write("delegate ");
stubWriter.Write(invokeMethod.ReturnType.GetQualifiedName());
stubWriter.Write($" {symbol.Name}");
StubTypeParameters(symbol.TypeParameters);
stubWriter.Write("(");
StubParameters(invokeMethod.Parameters);
stubWriter.Write(")");
StubTypeParameterConstraints(symbol.TypeParameters);
stubWriter.WriteLine(";");
return;
}
switch (symbol.TypeKind)
{
case TypeKind.Class:
StubAttributes(symbol.GetAttributes());
StubModifiers(symbol);
// certain classes, such as `Microsoft.Extensions.Logging.LoggingBuilderExtensions`
// exist in multiple assemblies, so make them partial
if (symbol.IsStatic && symbol.Name.EndsWith("Extensions"))
stubWriter.Write("partial ");
stubWriter.Write("class ");
break;
case TypeKind.Enum:
StubAttributes(symbol.GetAttributes());
StubModifiers(symbol);
stubWriter.Write("enum ");
break;
case TypeKind.Interface:
StubAttributes(symbol.GetAttributes());
StubModifiers(symbol);
stubWriter.Write("interface ");
break;
case TypeKind.Struct:
StubAttributes(symbol.GetAttributes());
StubModifiers(symbol);
stubWriter.Write("struct ");
break;
default:
return;
}
stubWriter.Write(symbol.Name);
StubTypeParameters(symbol.TypeParameters);
if (symbol.TypeKind == TypeKind.Enum)
{
if (symbol.EnumUnderlyingType is INamedTypeSymbol enumBase && enumBase.SpecialType != SpecialType.System_Int32)
{
stubWriter.Write(" : ");
stubWriter.Write(enumBase.GetQualifiedName());
}
}
else
{
var bases = symbol.Interfaces.Where(IsRelevantBaseType).OrderBy(i => i.GetName()).ToList();
if (GetBaseType(symbol) is INamedTypeSymbol @base && IsRelevantBaseType(@base))
{
bases.Insert(0, @base);
}
if (bases.Any())
{
stubWriter.Write(" : ");
WriteStringCommaSep(bases, b => b.GetQualifiedName());
}
}
StubTypeParameterConstraints(symbol.TypeParameters);
stubWriter.WriteLine(" {");
if (symbol.TypeKind == TypeKind.Enum)
{
foreach (var field in symbol.GetMembers().OfType<IFieldSymbol>().Where(field => field.ConstantValue is not null))
{
stubWriter.Write(field.Name);
stubWriter.Write(" = ");
stubWriter.Write(field.ConstantValue);
stubWriter.WriteLine(",");
}
}
else
{
var seenCtor = false;
foreach (var childSymbol in symbol.GetMembers().OrderBy(m => m.GetName()))
{
seenCtor |= childSymbol is IMethodSymbol method && method.MethodKind == MethodKind.Constructor;
childSymbol.Accept(this);
}
if (!seenCtor && GetBaseConstructor(symbol) is IMethodSymbol baseCtor)
{
stubWriter.Write($"internal {symbol.Name}() : base(");
WriteStringCommaSep(baseCtor.Parameters, parameter => $"default({parameter.Type.GetQualifiedName()})");
stubWriter.WriteLine(") {}");
}
}
stubWriter.WriteLine("}");
}
public override void VisitNamespace(INamespaceSymbol symbol)
{
if (!relevantSymbol.IsRelevantNamespace(symbol))
{
return;
}
var isGlobal = symbol.IsGlobalNamespace;
if (!isGlobal)
stubWriter.WriteLine($"namespace {symbol.Name} {{");
foreach (var childSymbol in symbol.GetMembers().OrderBy(m => m.GetName()))
{
childSymbol.Accept(this);
}
if (!isGlobal)
stubWriter.WriteLine("}");
}
private void StubProperty(IPropertySymbol symbol, IPropertySymbol? explicitInterfaceSymbol)
{
if (symbol.Parameters.Any())
{
var name = symbol.GetName(useMetadataName: true);
if (name is not "Item" && explicitInterfaceSymbol is null)
stubWriter.WriteLine($"[System.Runtime.CompilerServices.IndexerName(\"{name}\")]");
}
StubAttributes(symbol.GetAttributes());
StubModifiers(symbol, explicitInterfaceSymbol is not null);
if (IsUnsafe(symbol.Type) || symbol.Parameters.Any(p => IsUnsafe(p.Type)))
{
stubWriter.Write("unsafe ");
}
stubWriter.Write(symbol.Type.GetQualifiedName());
stubWriter.Write(" ");
if (symbol.Parameters.Any())
{
StubExplicitInterface(symbol, explicitInterfaceSymbol, writeName: false);
stubWriter.Write("this[");
StubParameters(symbol.Parameters);
stubWriter.Write("]");
}
else
{
StubExplicitInterface(symbol, explicitInterfaceSymbol);
}
stubWriter.Write(" { ");
if (symbol.GetMethod is not null)
stubWriter.Write(symbol.IsAbstract ? "get; " : "get => throw null; ");
if (symbol.SetMethod is not null)
stubWriter.Write(symbol.IsAbstract ? "set; " : "set {} ");
stubWriter.WriteLine("}");
}
public override void VisitProperty(IPropertySymbol symbol)
{
var explicitInterfaceImplementations = FilterExplicitInterfaceImplementations(symbol.ExplicitInterfaceImplementations);
if (IsNotPublic(symbol.DeclaredAccessibility) && explicitInterfaceImplementations.Length == 0)
return;
foreach (var explicitInterfaceImplementation in explicitInterfaceImplementations)
{
StubProperty(symbol, explicitInterfaceImplementation);
}
if (explicitInterfaceImplementations.Length == 0)
StubProperty(symbol, null);
}
}

9
csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/SymbolExtensions.cs Normal file
View File

@@ -0,0 +1,9 @@
using Microsoft.CodeAnalysis;
namespace Semmle.Extraction.CSharp.StubGenerator;
public static class SymbolExtensions
{
public static string GetQualifiedName(this ISymbol symbol) =>
symbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat.WithGlobalNamespaceStyle(SymbolDisplayGlobalNamespaceStyle.Omitted));
}

17
csharp/extractor/Semmle.Extraction.CSharp.Util/Semmle.Extraction.CSharp.Util.csproj Normal file
View File

@@ -0,0 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<AssemblyName>Semmle.Extraction.CSharp.Util</AssemblyName>
<RootNamespace>Semmle.Extraction.CSharp.Util</RootNamespace>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.4.0" />
</ItemGroup>
</Project>

131
csharp/extractor/Semmle.Extraction.CSharp.Util/SymbolExtensions.cs Normal file
View File

@@ -0,0 +1,131 @@
using System.Text.RegularExpressions;
using Microsoft.CodeAnalysis;
namespace Semmle.Extraction.CSharp.Util
{
public static partial class SymbolExtensions
{
/// <summary>
/// Gets the name of this symbol.
///
/// If the symbol implements an explicit interface, only the
/// name of the member being implemented is included, not the
/// explicit prefix.
/// </summary>
public static string GetName(this ISymbol symbol, bool useMetadataName = false)
{
var name = useMetadataName ? symbol.MetadataName : symbol.Name;
return symbol.CanBeReferencedByName ? name : name.Substring(symbol.Name.LastIndexOf('.') + 1);
}
/// <summary>
/// Convert an operator method name in to a symbolic name.
/// A return value indicates whether the conversion succeeded.
/// </summary>
public static bool TryGetOperatorSymbol(this ISymbol symbol, out string operatorName)
{
static bool TryGetOperatorSymbolFromName(string methodName, out string operatorName)
{
var success = true;
switch (methodName)
{
case "op_LogicalNot":
operatorName = "!";
break;
case "op_BitwiseAnd":
operatorName = "&";
break;
case "op_Equality":
operatorName = "==";
break;
case "op_Inequality":
operatorName = "!=";
break;
case "op_UnaryPlus":
case "op_Addition":
operatorName = "+";
break;
case "op_UnaryNegation":
case "op_Subtraction":
operatorName = "-";
break;
case "op_Multiply":
operatorName = "*";
break;
case "op_Division":
operatorName = "/";
break;
case "op_Modulus":
operatorName = "%";
break;
case "op_GreaterThan":
operatorName = ">";
break;
case "op_GreaterThanOrEqual":
operatorName = ">=";
break;
case "op_LessThan":
operatorName = "<";
break;
case "op_LessThanOrEqual":
operatorName = "<=";
break;
case "op_Decrement":
operatorName = "--";
break;
case "op_Increment":
operatorName = "++";
break;
case "op_Implicit":
operatorName = "implicit conversion";
break;
case "op_Explicit":
operatorName = "explicit conversion";
break;
case "op_OnesComplement":
operatorName = "~";
break;
case "op_RightShift":
operatorName = ">>";
break;
case "op_UnsignedRightShift":
operatorName = ">>>";
break;
case "op_LeftShift":
operatorName = "<<";
break;
case "op_BitwiseOr":
operatorName = "|";
break;
case "op_ExclusiveOr":
operatorName = "^";
break;
case "op_True":
operatorName = "true";
break;
case "op_False":
operatorName = "false";
break;
default:
var match = CheckedRegex().Match(methodName);
if (match.Success)
{
TryGetOperatorSymbolFromName("op_" + match.Groups[1], out var uncheckedName);
operatorName = "checked " + uncheckedName;
break;
}
operatorName = methodName;
success = false;
break;
}
return success;
}
var methodName = symbol.GetName(useMetadataName: false);
return TryGetOperatorSymbolFromName(methodName, out operatorName);
}
[GeneratedRegex("^op_Checked(.*)$")]
private static partial Regex CheckedRegex();
}
}

1
csharp/extractor/Semmle.Extraction.CSharp/Entities/Event.cs
View File

@@ -2,6 +2,7 @@ using System.IO;
using System.Linq;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp.Syntax;
using Semmle.Extraction.CSharp.Util;
namespace Semmle.Extraction.CSharp.Entities
{

5
csharp/extractor/Semmle.Extraction.CSharp/Entities/Expression.cs
View File

@@ -6,6 +6,7 @@ using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
using Microsoft.CodeAnalysis.CSharp.Syntax;
using Semmle.Extraction.CSharp.Entities.Expressions;
using Semmle.Extraction.CSharp.Util;
using Semmle.Extraction.Kinds;
namespace Semmle.Extraction.CSharp.Entities
@@ -205,7 +206,7 @@ namespace Semmle.Extraction.CSharp.Entities
{
// this can happen in VB.NET
cx.ExtractionError($"Extracting default argument value 'object {parameter.Name} = default' instead of 'object {parameter.Name} = {defaultValue}'. The latter is not supported in C#.",
null, null, severity: Util.Logging.Severity.Warning);
null, null, severity: Semmle.Util.Logging.Severity.Warning);
// we're generating a default expression:
return Default.CreateGenerated(cx, parent, childIndex, location, ValueAsString(null));
@@ -250,7 +251,7 @@ namespace Semmle.Extraction.CSharp.Entities
var callType = GetCallType(Context, node);
if (callType == CallType.Dynamic)
{
UserOperator.TryGetOperatorSymbol(method.Name, out var operatorName);
method.TryGetOperatorSymbol(out var operatorName);
trapFile.dynamic_member_name(this, operatorName);
return;
}

1
csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ImplicitCast.cs
View File

@@ -1,5 +1,6 @@
using System.Linq;
using Microsoft.CodeAnalysis;
using Semmle.Extraction.CSharp.Util;
using Semmle.Extraction.Kinds;
namespace Semmle.Extraction.CSharp.Entities.Expressions

1
csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ObjectCreation/DateTimeObjectCreation.cs
View File

@@ -1,6 +1,7 @@
using System.IO;
using System.Linq;
using Microsoft.CodeAnalysis;
using Semmle.Extraction.CSharp.Util;
using Semmle.Extraction.Kinds;
namespace Semmle.Extraction.CSharp.Entities.Expressions

1
csharp/extractor/Semmle.Extraction.CSharp/Entities/Indexer.cs
View File

@@ -2,6 +2,7 @@ using System.IO;
using System.Linq;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp.Syntax;
using Semmle.Extraction.CSharp.Util;
namespace Semmle.Extraction.CSharp.Entities
{

4
csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs
View File

@@ -3,7 +3,7 @@ using System.Linq;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp.Syntax;
using Semmle.Extraction.CSharp.Populators;
using Semmle.Extraction.CSharp.Util;
namespace Semmle.Extraction.CSharp.Entities
{
@@ -51,7 +51,7 @@ namespace Semmle.Extraction.CSharp.Entities
{
if (method.MethodKind == MethodKind.ReducedExtension)
{
cx.Extractor.Logger.Log(Util.Logging.Severity.Warning, "Reduced extension method symbols should not be directly extracted.");
cx.Extractor.Logger.Log(Semmle.Util.Logging.Severity.Warning, "Reduced extension method symbols should not be directly extracted.");
}
return OrdinaryMethodFactory.Instance.CreateEntityFromSymbol(cx, method);

1
csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs
View File

@@ -5,6 +5,7 @@ using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp.Syntax;
using Semmle.Extraction.CSharp.Entities.Expressions;
using Semmle.Extraction.Kinds;
using Semmle.Extraction.CSharp.Util;
namespace Semmle.Extraction.CSharp.Entities
{

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/ForEach.cs
View File

@@ -44,7 +44,7 @@ namespace Semmle.Extraction.CSharp.Entities.Statements
if (info.Equals(default))
{
Context.ExtractionError("Could not get foreach statement info", null, Context.CreateLocation(this.ReportingLocation), severity: Util.Logging.Severity.Info);
Context.ExtractionError("Could not get foreach statement info", null, Context.CreateLocation(this.ReportingLocation), severity: Semmle.Util.Logging.Severity.Info);
return;
}

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/LocalFunction.cs
View File

@@ -34,7 +34,7 @@ namespace Semmle.Extraction.CSharp.Entities.Statements
{
if (Symbol is null)
{
Context.ExtractionError("Could not get local function symbol", null, Context.CreateLocation(this.ReportingLocation), severity: Util.Logging.Severity.Warning);
Context.ExtractionError("Could not get local function symbol", null, Context.CreateLocation(this.ReportingLocation), severity: Semmle.Util.Logging.Severity.Warning);
return;
}

2
csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs
View File

@@ -87,7 +87,7 @@ namespace Semmle.Extraction.CSharp.Entities
var hasExpandingCycle = GenericsRecursionGraph.HasExpandingCycle(Symbol);
if (hasExpandingCycle)
{
Context.ExtractionError("Found recursive generic inheritance hierarchy. Base class of type is not extracted", Symbol.ToDisplayString(), Context.CreateLocation(ReportingLocation), severity: Util.Logging.Severity.Warning);
Context.ExtractionError("Found recursive generic inheritance hierarchy. Base class of type is not extracted", Symbol.ToDisplayString(), Context.CreateLocation(ReportingLocation), severity: Semmle.Util.Logging.Severity.Warning);
}
// Visit base types

113
csharp/extractor/Semmle.Extraction.CSharp/Entities/UserOperator.cs
View File

@@ -1,8 +1,8 @@
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp.Syntax;
using Semmle.Extraction.CSharp.Util;
namespace Semmle.Extraction.CSharp.Entities
{
@@ -79,108 +79,7 @@ namespace Semmle.Extraction.CSharp.Entities
return true;
}
/// <summary>
/// Convert an operator method name in to a symbolic name.
/// A return value indicates whether the conversion succeeded.
/// </summary>
/// <param name="methodName">The method name.</param>
/// <param name="operatorName">The converted operator name.</param>
public static bool TryGetOperatorSymbol(string methodName, out string operatorName)
{
var success = true;
switch (methodName)
{
case "op_LogicalNot":
operatorName = "!";
break;
case "op_BitwiseAnd":
operatorName = "&";
break;
case "op_Equality":
operatorName = "==";
break;
case "op_Inequality":
operatorName = "!=";
break;
case "op_UnaryPlus":
case "op_Addition":
operatorName = "+";
break;
case "op_UnaryNegation":
case "op_Subtraction":
operatorName = "-";
break;
case "op_Multiply":
operatorName = "*";
break;
case "op_Division":
operatorName = "/";
break;
case "op_Modulus":
operatorName = "%";
break;
case "op_GreaterThan":
operatorName = ">";
break;
case "op_GreaterThanOrEqual":
operatorName = ">=";
break;
case "op_LessThan":
operatorName = "<";
break;
case "op_LessThanOrEqual":
operatorName = "<=";
break;
case "op_Decrement":
operatorName = "--";
break;
case "op_Increment":
operatorName = "++";
break;
case "op_Implicit":
operatorName = "implicit conversion";
break;
case "op_Explicit":
operatorName = "explicit conversion";
break;
case "op_OnesComplement":
operatorName = "~";
break;
case "op_RightShift":
operatorName = ">>";
break;
case "op_UnsignedRightShift":
operatorName = ">>>";
break;
case "op_LeftShift":
operatorName = "<<";
break;
case "op_BitwiseOr":
operatorName = "|";
break;
case "op_ExclusiveOr":
operatorName = "^";
break;
case "op_True":
operatorName = "true";
break;
case "op_False":
operatorName = "false";
break;
default:
var match = Regex.Match(methodName, "^op_Checked(.*)$");
if (match.Success)
{
TryGetOperatorSymbol("op_" + match.Groups[1], out var uncheckedName);
operatorName = "checked " + uncheckedName;
break;
}
operatorName = methodName;
success = false;
break;
}
return success;
}
/// <summary>
/// Converts a method name into a symbolic name.
@@ -191,12 +90,8 @@ namespace Semmle.Extraction.CSharp.Entities
/// <returns>The converted name.</returns>
private static string OperatorSymbol(Context cx, IMethodSymbol method)
{
if (method.ExplicitInterfaceImplementations.Any())
return OperatorSymbol(cx, method.ExplicitInterfaceImplementations.First());
var methodName = method.Name;
if (!TryGetOperatorSymbol(methodName, out var result))
cx.ModelError(method, $"Unhandled operator name in OperatorSymbol(): '{methodName}'");
if (!method.TryGetOperatorSymbol(out var result))
cx.ModelError(method, $"Unhandled operator name in OperatorSymbol(): '{method.Name}'");
return result;
}

4
csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs
View File

@@ -71,9 +71,9 @@ namespace Semmle.Extraction.CSharp
public static ILogger MakeLogger(Verbosity verbosity, bool includeConsole)
{
var fileLogger = new FileLogger(verbosity, GetCSharpLogPath());
var fileLogger = new FileLogger(verbosity, GetCSharpLogPath(), logThreadId: true);
return includeConsole
? new CombinedLogger(new ConsoleLogger(verbosity), fileLogger)
? new CombinedLogger(new ConsoleLogger(verbosity, logThreadId: true), fileLogger)
: (ILogger)fileLogger;
}

8
csharp/extractor/Semmle.Extraction.CSharp/Populators/DirectiveVisitor.cs
View File

@@ -59,7 +59,7 @@ namespace Semmle.Extraction.CSharp.Populators
if (regionStarts.Count == 0)
{
cx.ExtractionError("Couldn't find start region", null,
cx.CreateLocation(node.GetLocation()), null, Util.Logging.Severity.Warning);
cx.CreateLocation(node.GetLocation()), null, Semmle.Util.Logging.Severity.Warning);
return;
}
@@ -94,7 +94,7 @@ namespace Semmle.Extraction.CSharp.Populators
if (ifStarts.Count == 0)
{
cx.ExtractionError("Couldn't find start if", null,
cx.CreateLocation(node.GetLocation()), null, Util.Logging.Severity.Warning);
cx.CreateLocation(node.GetLocation()), null, Semmle.Util.Logging.Severity.Warning);
return;
}
@@ -107,7 +107,7 @@ namespace Semmle.Extraction.CSharp.Populators
if (ifStarts.Count == 0)
{
cx.ExtractionError("Couldn't find start if", null,
cx.CreateLocation(node.GetLocation()), null, Util.Logging.Severity.Warning);
cx.CreateLocation(node.GetLocation()), null, Semmle.Util.Logging.Severity.Warning);
return;
}
@@ -122,7 +122,7 @@ namespace Semmle.Extraction.CSharp.Populators
if (ifStarts.Count == 0)
{
cx.ExtractionError("Couldn't find start if", null,
cx.CreateLocation(node.GetLocation()), null, Util.Logging.Severity.Warning);
cx.CreateLocation(node.GetLocation()), null, Semmle.Util.Logging.Severity.Warning);
return;
}

2
csharp/extractor/Semmle.Extraction.CSharp/Semmle.Extraction.CSharp.csproj
View File

@@ -6,12 +6,12 @@
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Semmle.Extraction.CIL\Semmle.Extraction.CIL.csproj" />
<ProjectReference Include="..\Semmle.Extraction\Semmle.Extraction.csproj" />
<ProjectReference Include="..\Semmle.Extraction.CSharp.Util\Semmle.Extraction.CSharp.Util.csproj" />
<ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
</ItemGroup>
<ItemGroup>

13
csharp/extractor/Semmle.Extraction.CSharp/SymbolExtensions.cs
View File

@@ -55,19 +55,6 @@ namespace Semmle.Extraction.CSharp
: type;
}
/// <summary>
/// Gets the name of this symbol.
///
/// If the symbol implements an explicit interface, only the
/// name of the member being implemented is included, not the
/// explicit prefix.
/// </summary>
public static string GetName(this ISymbol symbol, bool useMetadataName = false)
{
var name = useMetadataName ? symbol.MetadataName : symbol.Name;
return symbol.CanBeReferencedByName ? name : name.Substring(symbol.Name.LastIndexOf('.') + 1);
}
private static IEnumerable<SyntaxToken> GetModifiers<T>(this ISymbol symbol, Func<T, IEnumerable<SyntaxToken>> getModifierTokens) =>
symbol.DeclaringSyntaxReferences
.Select(r => r.GetSyntax())

239
csharp/extractor/Semmle.Extraction.Tests/DotNet.cs Normal file
View File

@@ -0,0 +1,239 @@
using Xunit;
using System.Collections.Generic;
using Semmle.Extraction.CSharp.DependencyFetching;
using System;
using System.Linq;
namespace Semmle.Extraction.Tests
{
internal class DotNetCliInvokerStub : IDotNetCliInvoker
{
private readonly IList<string> output;
private string lastArgs = "";
public bool Success { get; set; } = true;
public DotNetCliInvokerStub(IList<string> output)
{
this.output = output;
}
public string Exec => "dotnet";
public bool RunCommand(string args)
{
lastArgs = args;
return Success;
}
public bool RunCommand(string args, out IList<string> output)
{
lastArgs = args;
output = this.output;
return Success;
}
public string GetLastArgs() => lastArgs;
}
public class DotNetTests
{
private static IDotNet MakeDotnet(IDotNetCliInvoker dotnetCliInvoker) =>
DotNet.Make(dotnetCliInvoker, new ProgressMonitor(new LoggerStub()));
private static IList<string> MakeDotnetRestoreOutput() =>
new List<string> {
" Determining projects to restore...",
" Restored /path/to/project.csproj (in 1.23 sec).",
" Other output...",
" More output...",
" Restored /path/to/project2.csproj (in 4.56 sec).",
" Other output...",
};
private static IList<string> MakeDotnetListRuntimesOutput() =>
new List<string> {
"Microsoft.AspNetCore.App 7.0.2 [/path/dotnet/shared/Microsoft.AspNetCore.App]",
"Microsoft.NETCore.App 7.0.2 [/path/dotnet/shared/Microsoft.NETCore.App]"
};
[Fact]
public void TestDotnetInfo()
{
// Setup
var dotnetCliInvoker = new DotNetCliInvokerStub(new List<string>());
// Execute
var _ = MakeDotnet(dotnetCliInvoker);
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("--info", lastArgs);
}
[Fact]
public void TestDotnetInfoFailure()
{
// Setup
var dotnetCliInvoker = new DotNetCliInvokerStub(new List<string>()) { Success = false };
// Execute
try
{
var _ = MakeDotnet(dotnetCliInvoker);
}
// Verify
catch (Exception e)
{
Assert.Equal("dotnet --info failed.", e.Message);
return;
}
Assert.Fail("Expected exception");
}
[Fact]
public void TestDotnetRestoreProjectToDirectory1()
{
// Setup
var dotnetCliInvoker = new DotNetCliInvokerStub(new List<string>());
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages");
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true", lastArgs);
}
[Fact]
public void TestDotnetRestoreProjectToDirectory2()
{
// Setup
var dotnetCliInvoker = new DotNetCliInvokerStub(new List<string>());
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", "myconfig.config");
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --configfile \"myconfig.config\"", lastArgs);
}
[Fact]
public void TestDotnetRestoreSolutionToDirectory1()
{
// Setup
var dotnetCliInvoker = new DotNetCliInvokerStub(MakeDotnetRestoreOutput());
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.RestoreSolutionToDirectory("mysolution.sln", "mypackages", out var projects);
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"mysolution.sln\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal", lastArgs);
Assert.Equal(2, projects.Count());
Assert.Contains("/path/to/project.csproj", projects);
Assert.Contains("/path/to/project2.csproj", projects);
}
[Fact]
public void TestDotnetRestoreSolutionToDirectory2()
{
// Setup
var dotnetCliInvoker = new DotNetCliInvokerStub(MakeDotnetRestoreOutput());
var dotnet = MakeDotnet(dotnetCliInvoker);
dotnetCliInvoker.Success = false;
// Execute
dotnet.RestoreSolutionToDirectory("mysolution.sln", "mypackages", out var projects);
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"mysolution.sln\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal", lastArgs);
Assert.Empty(projects);
}
[Fact]
public void TestDotnetNew()
{
// Setup
var dotnetCliInvoker = new DotNetCliInvokerStub(new List<string>());
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.New("myfolder");
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("new console --no-restore --output \"myfolder\"", lastArgs);
}
[Fact]
public void TestDotnetAddPackage()
{
// Setup
var dotnetCliInvoker = new DotNetCliInvokerStub(new List<string>());
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.AddPackage("myfolder", "mypackage");
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("add \"myfolder\" package \"mypackage\" --no-restore", lastArgs);
}
[Fact]
public void TestDotnetGetListedRuntimes1()
{
// Setup
var dotnetCliInvoker = new DotNetCliInvokerStub(MakeDotnetListRuntimesOutput());
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
var runtimes = dotnet.GetListedRuntimes();
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("--list-runtimes", lastArgs);
Assert.Equal(2, runtimes.Count);
Assert.Contains("Microsoft.AspNetCore.App 7.0.2 [/path/dotnet/shared/Microsoft.AspNetCore.App]", runtimes);
Assert.Contains("Microsoft.NETCore.App 7.0.2 [/path/dotnet/shared/Microsoft.NETCore.App]", runtimes);
}
[Fact]
public void TestDotnetGetListedRuntimes2()
{
// Setup
var dotnetCliInvoker = new DotNetCliInvokerStub(MakeDotnetListRuntimesOutput());
var dotnet = MakeDotnet(dotnetCliInvoker);
dotnetCliInvoker.Success = false;
// Execute
var runtimes = dotnet.GetListedRuntimes();
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("--list-runtimes", lastArgs);
Assert.Empty(runtimes);
}
[Fact]
public void TestDotnetExec()
{
// Setup
var dotnetCliInvoker = new DotNetCliInvokerStub(new List<string>());
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.Exec("myarg1 myarg2");
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("exec myarg1 myarg2", lastArgs);
}
}
}

64
csharp/extractor/Semmle.Extraction.Tests/FileContent.cs
View File

@@ -1,18 +1,9 @@
using Xunit;
using System.Collections.Generic;
using Semmle.Util.Logging;
using Semmle.Extraction.CSharp.DependencyFetching;
namespace Semmle.Extraction.Tests
{
internal class LoggerStub : ILogger
{
public void Log(Severity severity, string message) { }
public void Dispose() { }
}
internal class UnsafeFileReaderStub : IUnsafeFileReader
{
private readonly List<string> lines;
@@ -90,5 +81,60 @@ namespace Semmle.Extraction.Tests
Assert.Contains("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), allPackages);
Assert.Contains("StyleCop.Analyzers".ToLowerInvariant(), allPackages);
}
private static void ImplicitUsingsTest(string line, bool expected)
{
// Setup
var lines = new List<string>()
{
line
};
var fileContent = new TestFileContent(lines);
// Execute
var useImplicitUsings = fileContent.UseImplicitUsings;
// Verify
Assert.Equal(expected, useImplicitUsings);
}
[Fact]
public void TestFileContent_ImplicitUsings0()
{
ImplicitUsingsTest("<ImplicitUsings>false</ImplicitUsings>", false);
}
[Fact]
public void TestFileContent_ImplicitUsings1()
{
ImplicitUsingsTest("<ImplicitUsings>true</ImplicitUsings>", true);
}
[Fact]
public void TestFileContent_ImplicitUsings2()
{
ImplicitUsingsTest("<ImplicitUsings>enable</ImplicitUsings>", true);
}
[Fact]
public void TestFileContent_ImplicitUsingsAdditional()
{
// Setup
var lines = new List<string>()
{
"<Using Include=\"Ns0.Ns1\" />",
"<Using Include=\"Ns2\" />",
"<Using Remove=\"Ns3\" />",
};
var fileContent = new TestFileContent(lines);
// Execute
var customImplicitUsings = fileContent.CustomImplicitUsings;
// Verify
Assert.Equal(2, customImplicitUsings.Count);
Assert.Contains("Ns0.Ns1", customImplicitUsings);
Assert.Contains("Ns2", customImplicitUsings);
}
}
}

11
csharp/extractor/Semmle.Extraction.Tests/LoggerStub.cs Normal file
View File

@@ -0,0 +1,11 @@
using Semmle.Util.Logging;
namespace Semmle.Extraction.Tests
{
internal class LoggerStub : ILogger
{
public void Log(Severity severity, string message) { }
public void Dispose() { }
}
}

6
csharp/extractor/Semmle.Extraction.Tests/Runtime.cs
View File

@@ -19,11 +19,7 @@ namespace Semmle.Extraction.Tests
public bool New(string folder) => true;
public bool RestoreProjectToDirectory(string project, string directory, out string stdout, string? pathToNugetConfig = null)
{
stdout = "";
return true;
}
public bool RestoreProjectToDirectory(string project, string directory, string? pathToNugetConfig = null) => true;
public bool RestoreSolutionToDirectory(string solution, string directory, out IEnumerable<string> projects)
{

52
csharp/extractor/Semmle.Extraction.Tests/TrapWriter.cs
View File

@@ -1,52 +0,0 @@
using Xunit;
using Semmle.Util.Logging;
using Semmle.Util;
namespace Semmle.Extraction.Tests
{
public class TrapWriterTests
{
[Fact]
public void NestedPaths()
{
string tempDir = System.IO.Path.GetTempPath();
string root1, root2, root3;
if (Win32.IsWindows())
{
root1 = "E:";
root2 = "e:";
root3 = @"\";
}
else
{
root1 = "/E_";
root2 = "/e_";
root3 = "/";
}
using var logger = new LoggerMock();
Assert.Equal($@"C:\Temp\source_archive\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", "def.cs").Replace('/', '\\'));
Assert.Equal(@"C:\Temp\source_archive\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", "def.cs").Replace('/', '\\'));
Assert.Equal(@"C:\Temp\source_archive\E_\source\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", $@"{root1}\source\def.cs").Replace('/', '\\'));
Assert.Equal(@"C:\Temp\source_archive\e_\source\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", $@"{root2}\source\def.cs").Replace('/', '\\'));
Assert.Equal(@"C:\Temp\source_archive\source\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", $@"{root3}source\def.cs").Replace('/', '\\'));
Assert.Equal(@"C:\Temp\source_archive\source\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", $@"{root3}source\def.cs").Replace('/', '\\'));
Assert.Equal(@"C:\Temp\source_archive\diskstation\share\source\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", $@"{root3}{root3}diskstation\share\source\def.cs").Replace('/', '\\'));
}
private sealed class LoggerMock : ILogger
{
public void Dispose() { }
public void Log(Severity s, string text) { }
}
}
}

36
csharp/extractor/Semmle.Extraction/TrapWriter.cs
View File

@@ -216,7 +216,7 @@ namespace Semmle.Extraction
private void ArchiveContents(PathTransformer.ITransformedPath transformedPath, string contents)
{
var dest = NestPaths(logger, archive, transformedPath.Value);
var dest = FileUtils.NestPaths(logger, archive, transformedPath.Value);
var tmpSrcFile = Path.GetTempFileName();
File.WriteAllText(tmpSrcFile, contents, utf8);
try
@@ -231,38 +231,6 @@ namespace Semmle.Extraction
}
}
public static string NestPaths(ILogger logger, string? outerpath, string innerpath)
{
var nested = innerpath;
if (!string.IsNullOrEmpty(outerpath))
{
// Remove all leading path separators / or \
// For example, UNC paths have two leading \\
innerpath = innerpath.TrimStart(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
if (innerpath.Length > 1 && innerpath[1] == ':')
innerpath = innerpath[0] + "_" + innerpath.Substring(2);
nested = Path.Combine(outerpath, innerpath);
}
try
{
var directoryName = Path.GetDirectoryName(nested);
if (directoryName is null)
{
logger.Log(Severity.Warning, "Failed to get directory name from path '" + nested + "'.");
throw new InvalidOperationException();
}
Directory.CreateDirectory(directoryName);
}
catch (PathTooLongException)
{
logger.Log(Severity.Warning, "Failed to create parent directory of '" + nested + "': Path too long.");
throw;
}
return nested;
}
private static string TrapExtension(CompressionMode compression)
{
switch (compression)
@@ -280,7 +248,7 @@ namespace Semmle.Extraction
if (string.IsNullOrEmpty(folder))
folder = Directory.GetCurrentDirectory();
return NestPaths(logger, folder, filename);
return FileUtils.NestPaths(logger, folder, filename);
}
}
}

43
csharp/extractor/Semmle.Util.Tests/FileUtils.cs
View File

@@ -1,5 +1,6 @@
using Xunit;
using Semmle.Util;
using Semmle.Util.Logging;
namespace SemmleTests.Semmle.Util
{
@@ -16,5 +17,47 @@ namespace SemmleTests.Semmle.Util
Assert.Equal(Win32.IsWindows() ? @"foo\bar" : "foo/bar", FileUtils.ConvertToNative("foo/bar"));
}
[Fact]
public void NestedPaths()
{
string root1, root2, root3;
if (Win32.IsWindows())
{
root1 = "E:";
root2 = "e:";
root3 = @"\";
}
else
{
root1 = "/E_";
root2 = "/e_";
root3 = "/";
}
using var logger = new LoggerMock();
Assert.Equal($@"C:\Temp\source_archive\def.cs", FileUtils.NestPaths(logger, @"C:\Temp\source_archive", "def.cs").Replace('/', '\\'));
Assert.Equal(@"C:\Temp\source_archive\def.cs", FileUtils.NestPaths(logger, @"C:\Temp\source_archive", "def.cs").Replace('/', '\\'));
Assert.Equal(@"C:\Temp\source_archive\E_\source\def.cs", FileUtils.NestPaths(logger, @"C:\Temp\source_archive", $@"{root1}\source\def.cs").Replace('/', '\\'));
Assert.Equal(@"C:\Temp\source_archive\e_\source\def.cs", FileUtils.NestPaths(logger, @"C:\Temp\source_archive", $@"{root2}\source\def.cs").Replace('/', '\\'));
Assert.Equal(@"C:\Temp\source_archive\source\def.cs", FileUtils.NestPaths(logger, @"C:\Temp\source_archive", $@"{root3}source\def.cs").Replace('/', '\\'));
Assert.Equal(@"C:\Temp\source_archive\source\def.cs", FileUtils.NestPaths(logger, @"C:\Temp\source_archive", $@"{root3}source\def.cs").Replace('/', '\\'));
Assert.Equal(@"C:\Temp\source_archive\diskstation\share\source\def.cs", FileUtils.NestPaths(logger, @"C:\Temp\source_archive", $@"{root3}{root3}diskstation\share\source\def.cs").Replace('/', '\\'));
}
private sealed class LoggerMock : ILogger
{
public void Dispose() { }
public void Log(Severity s, string text) { }
}
}
}

2
csharp/extractor/Semmle.Util/EnvironmentVariables.cs
View File

@@ -7,6 +7,8 @@ namespace Semmle.Util
public static string? GetExtractorOption(string name) =>
Environment.GetEnvironmentVariable($"CODEQL_EXTRACTOR_CSHARP_OPTION_{name.ToUpper()}");
public static string? GetScratchDirectory() => Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR");
public static int GetDefaultNumberOfThreads()
{
if (!int.TryParse(Environment.GetEnvironmentVariable("CODEQL_THREADS"), out var threads) || threads == -1)

33
csharp/extractor/Semmle.Util/FileUtils.cs
View File

@@ -5,6 +5,7 @@ using System.Net.Http;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using Semmle.Util.Logging;
namespace Semmle.Util
{
@@ -110,5 +111,37 @@ namespace Semmle.Util
/// </summary>
public static void DownloadFile(string address, string fileName) =>
DownloadFileAsync(address, fileName).Wait();
public static string NestPaths(ILogger logger, string? outerpath, string innerpath)
{
var nested = innerpath;
if (!string.IsNullOrEmpty(outerpath))
{
// Remove all leading path separators / or \
// For example, UNC paths have two leading \\
innerpath = innerpath.TrimStart(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
if (innerpath.Length > 1 && innerpath[1] == ':')
innerpath = innerpath[0] + "_" + innerpath.Substring(2);
nested = Path.Combine(outerpath, innerpath);
}
try
{
var directoryName = Path.GetDirectoryName(nested);
if (directoryName is null)
{
logger.Log(Severity.Warning, "Failed to get directory name from path '" + nested + "'.");
throw new InvalidOperationException();
}
Directory.CreateDirectory(directoryName);
}
catch (PathTooLongException)
{
logger.Log(Severity.Warning, "Failed to create parent directory of '" + nested + "': Path too long.");
throw;
}
return nested;
}
}
}

18
csharp/extractor/Semmle.Util/Logger.cs
View File

@@ -59,10 +59,12 @@ namespace Semmle.Util.Logging
{
private readonly StreamWriter writer;
private readonly Verbosity verbosity;
private readonly bool logThreadId;
public FileLogger(Verbosity verbosity, string outputFile)
public FileLogger(Verbosity verbosity, string outputFile, bool logThreadId)
{
this.verbosity = verbosity;
this.logThreadId = logThreadId;
try
{
@@ -93,7 +95,10 @@ namespace Semmle.Util.Logging
public void Log(Severity s, string text)
{
if (verbosity.Includes(s))
writer.WriteLine(GetSeverityPrefix(s) + text);
{
var threadId = this.logThreadId ? $"[{Environment.CurrentManagedThreadId:D3}] " : "";
writer.WriteLine(threadId + GetSeverityPrefix(s) + text);
}
}
}
@@ -103,10 +108,12 @@ namespace Semmle.Util.Logging
public sealed class ConsoleLogger : ILogger
{
private readonly Verbosity verbosity;
private readonly bool logThreadId;
public ConsoleLogger(Verbosity verbosity)
public ConsoleLogger(Verbosity verbosity, bool logThreadId)
{
this.verbosity = verbosity;
this.logThreadId = logThreadId;
}
public void Dispose() { }
@@ -136,7 +143,10 @@ namespace Semmle.Util.Logging
public void Log(Severity s, string text)
{
if (verbosity.Includes(s))
GetConsole(s).WriteLine(GetSeverityPrefix(s) + text);
{
var threadId = this.logThreadId ? $"[{Environment.CurrentManagedThreadId:D3}] " : "";
GetConsole(s).WriteLine(threadId + GetSeverityPrefix(s) + text);
}
}
}

39
csharp/extractor/Semmle.Util/MemoizedFunc.cs Normal file
View File

@@ -0,0 +1,39 @@
using System;
using System.Collections.Generic;
using System.Collections.Concurrent;
namespace Semmle.Util;
public class MemoizedFunc<T1, T2> where T1 : notnull
{
private readonly Func<T1, T2> f;
private readonly Dictionary<T1, T2> cache = new();
public MemoizedFunc(Func<T1, T2> f)
{
this.f = f;
}
public T2 Invoke(T1 s)
{
if (!cache.TryGetValue(s, out var t))
{
t = f(s);
cache[s] = t;
}
return t;
}
}
public class ConcurrentMemoizedFunc<T1, T2> where T1 : notnull
{
private readonly Func<T1, T2> f;
private readonly ConcurrentDictionary<T1, T2> cache = new();
public ConcurrentMemoizedFunc(Func<T1, T2> f)
{
this.f = f;
}
public T2 Invoke(T1 s) => cache.GetOrAdd(s, f);
}

56
csharp/extractor/Semmle.Util/ProcessStartInfoExtensions.cs
View File

@@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Diagnostics;
namespace Semmle.Util
@@ -9,25 +10,54 @@ namespace Semmle.Util
/// Runs this process, and returns the exit code, as well as the contents
/// of stdout in <paramref name="stdout"/>.
/// </summary>
public static int ReadOutput(this ProcessStartInfo pi, out IList<string> stdout)
public static int ReadOutput(this ProcessStartInfo pi, out IList<string> stdout, Action<string>? onOut, Action<string>? onError)
{
stdout = new List<string>();
using var process = Process.Start(pi);
if (process is null)
var @out = new List<string>();
using var process = new Process
{
return -1;
StartInfo = pi
};
if (process.StartInfo.RedirectStandardOutput && !pi.UseShellExecute)
{
process.OutputDataReceived += new DataReceivedEventHandler((sender, e) =>
{
if (e.Data == null)
{
return;
}
onOut?.Invoke(e.Data);
@out.Add(e.Data);
});
}
if (process.StartInfo.RedirectStandardError && !pi.UseShellExecute)
{
process.ErrorDataReceived += new DataReceivedEventHandler((sender, e) =>
{
if (e.Data == null)
{
return;
}
onError?.Invoke(e.Data);
});
}
string? s;
do
process.Start();
if (process.StartInfo.RedirectStandardError)
{
s = process.StandardOutput.ReadLine();
if (s is not null)
stdout.Add(s);
process.BeginErrorReadLine();
}
while (s is not null);
if (process.StartInfo.RedirectStandardOutput)
{
process.BeginOutputReadLine();
}
process.WaitForExit();
stdout = @out;
return process.ExitCode;
}
}

1
csharp/ql/integration-tests/all-platforms/cshtml_standalone/Files.expected
View File

@@ -1,3 +1,4 @@
| Program.cs |
| Views/Home/Index.cshtml |
| _semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_Views_Home_Index_cshtml.g.cs |
| test-db/working/implicitUsings/GlobalUsings.g.cs |

11
csharp/ql/integration-tests/all-platforms/cshtml_standalone/Files.ql
View File

@@ -1,14 +1,17 @@
import csharp
private string getPath(File f) {
result = f.getRelativePath()
result = f.getRelativePath() and
not exists(
result
.indexOf("_semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_")
)
or
not exists(f.getRelativePath()) and
exists(int index |
index =
f.getBaseName()
f.getRelativePath()
.indexOf("_semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_") and
result = f.getBaseName().substring(index, f.getBaseName().length())
result = f.getRelativePath().substring(index, f.getRelativePath().length())
)
}

5
csharp/ql/integration-tests/all-platforms/dotnet_run/test.py
View File

@@ -58,3 +58,8 @@ check_diagnostics(test_db="test8-db")
s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test8-db', 'dotnet run "hello world part1" part2'], "test9-db")
check_build_out("hello world part1, part2", s)
check_diagnostics(test_db="test9-db")
# two arguments, no '--' (second argument quoted) and using dotnet to execute dotnet
s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test9-db', 'dotnet dotnet run part1 "hello world part2"'], "test10-db")
check_build_out("part1, hello world part2", s)
check_diagnostics(test_db="test10-db")

1
csharp/ql/integration-tests/all-platforms/standalone/Files.expected
View File

@@ -1 +1,2 @@
| Program.cs:0:0:0:0 | Program.cs |
| test-db/working/implicitUsings/GlobalUsings.g.cs:0:0:0:0 | test-db/working/implicitUsings/GlobalUsings.g.cs |

1
csharp/ql/lib/ext/System.Collections.Generic.model.yml
View File

@@ -47,6 +47,7 @@ extensions:
- ["System.Collections.Generic", "List<>", False, "FindAll", "(System.Predicate<T>)", "", "Argument[this].Element", "ReturnValue", "value", "manual"]
- ["System.Collections.Generic", "List<>", False, "FindLast", "(System.Predicate<T>)", "", "Argument[this].Element", "Argument[0].Parameter[0]", "value", "manual"]
- ["System.Collections.Generic", "List<>", False, "FindLast", "(System.Predicate<T>)", "", "Argument[this].Element", "ReturnValue", "value", "manual"]
- ["System.Collections.Generic", "List<>", False, "ForEach", "(System.Action<T>)", "", "Argument[this].Element", "Argument[0].Parameter[0]", "value", "manual"]
- ["System.Collections.Generic", "List<>", False, "GetEnumerator", "()", "", "Argument[this].Element", "ReturnValue.Property[System.Collections.Generic.List<>+Enumerator.Current]", "value", "manual"]
- ["System.Collections.Generic", "List<>", False, "GetRange", "(System.Int32,System.Int32)", "", "Argument[this].Element", "ReturnValue.Element", "value", "manual"]
- ["System.Collections.Generic", "List<>", False, "InsertRange", "(System.Int32,System.Collections.Generic.IEnumerable<T>)", "", "Argument[1].Element", "Argument[this].Element", "value", "manual"]

45
csharp/ql/lib/semmle/code/csharp/dataflow/FlowSummary.qll
View File

@@ -4,6 +4,7 @@ import csharp
private import dotnet
private import internal.FlowSummaryImpl as Impl
private import internal.DataFlowDispatch as DataFlowDispatch
private import Impl::Public::SummaryComponent as SummaryComponentInternal
class ParameterPosition = DataFlowDispatch::ParameterPosition;
@@ -18,8 +19,6 @@ class SummaryComponent = Impl::Public::SummaryComponent;
/** Provides predicates for constructing summary components. */
module SummaryComponent {
private import Impl::Public::SummaryComponent as SummaryComponentInternal
predicate content = SummaryComponentInternal::content/1;
/** Gets a summary component for parameter `i`. */
@@ -155,3 +154,45 @@ private class RecordConstructorFlowRequiredSummaryComponentStack extends Require
)
}
}
class Provenance = Impl::Public::Provenance;
private import semmle.code.csharp.frameworks.system.linq.Expressions
private SummaryComponent delegateSelf() {
exists(ArgumentPosition pos |
result = SummaryComponentInternal::parameter(pos) and
pos.isDelegateSelf()
)
}
private predicate mayInvokeCallback(Callable c, int n) {
c.getParameter(n).getType() instanceof SystemLinqExpressions::DelegateExtType and
not c.fromSource()
}
private class SummarizedCallableWithCallback extends SummarizedCallable {
private int pos;
SummarizedCallableWithCallback() { mayInvokeCallback(this, pos) }
override predicate propagatesFlow(
SummaryComponentStack input, SummaryComponentStack output, boolean preservesValue
) {
input = SummaryComponentStack::argument(pos) and
output = SummaryComponentStack::push(delegateSelf(), input) and
preservesValue = true
}
override predicate hasProvenance(Provenance provenance) { provenance = "hq-generated" }
}
private class RequiredComponentStackForCallback extends RequiredSummaryComponentStack {
override predicate required(SummaryComponent head, SummaryComponentStack tail) {
exists(int pos |
mayInvokeCallback(_, pos) and
head = delegateSelf() and
tail = SummaryComponentStack::argument(pos)
)
}
}

40
csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowDispatch.qll
View File

@@ -136,13 +136,15 @@ private module Cached {
newtype TParameterPosition =
TPositionalParameterPosition(int i) { i = any(Parameter p).getPosition() } or
TThisParameterPosition() or
TImplicitCapturedParameterPosition(LocalScopeVariable v) { capturedWithFlowIn(v) }
TImplicitCapturedParameterPosition(LocalScopeVariable v) { capturedWithFlowIn(v) } or
TDelegateSelfParameterPosition()
cached
newtype TArgumentPosition =
TPositionalArgumentPosition(int i) { i = any(Parameter p).getPosition() } or
TQualifierArgumentPosition() or
TImplicitCapturedArgumentPosition(LocalScopeVariable v) { capturedWithFlowIn(v) }
TImplicitCapturedArgumentPosition(LocalScopeVariable v) { capturedWithFlowIn(v) } or
TDelegateSelfArgumentPosition()
}
import Cached
@@ -480,6 +482,14 @@ class ParameterPosition extends TParameterPosition {
this = TImplicitCapturedParameterPosition(v)
}
/**
* Holds if this position represents a reference to a delegate itself.
*
* Used for tracking flow through captured variables and for improving
* delegate dispatch.
*/
predicate isDelegateSelf() { this = TDelegateSelfParameterPosition() }
/** Gets a textual representation of this position. */
string toString() {
result = "position " + this.getPosition()
@@ -489,6 +499,9 @@ class ParameterPosition extends TParameterPosition {
exists(LocalScopeVariable v |
this.isImplicitCapturedParameterPosition(v) and result = "captured " + v
)
or
this.isDelegateSelf() and
result = "delegate self"
}
}
@@ -505,6 +518,14 @@ class ArgumentPosition extends TArgumentPosition {
this = TImplicitCapturedArgumentPosition(v)
}
/**
* Holds if this position represents a reference to a delegate itself.
*
* Used for tracking flow through captured variables and for improving
* delegate dispatch.
*/
predicate isDelegateSelf() { this = TDelegateSelfArgumentPosition() }
/** Gets a textual representation of this position. */
string toString() {
result = "position " + this.getPosition()
@@ -514,6 +535,9 @@ class ArgumentPosition extends TArgumentPosition {
exists(LocalScopeVariable v |
this.isImplicitCapturedArgumentPosition(v) and result = "captured " + v
)
or
this.isDelegateSelf() and
result = "delegate self"
}
}
@@ -527,14 +551,6 @@ predicate parameterMatch(ParameterPosition ppos, ArgumentPosition apos) {
ppos.isImplicitCapturedParameterPosition(v) and
apos.isImplicitCapturedArgumentPosition(v)
)
}
/**
* Holds if flow from `call`'s argument `arg` to parameter `p` is permissible.
*
* This is a temporary hook to support technical debt in the Go language; do not use.
*/
pragma[inline]
predicate golangSpecificParamArgFilter(DataFlowCall call, ParameterNode p, ArgumentNode arg) {
any()
or
ppos.isDelegateSelf() and apos.isDelegateSelf()
}

294
csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowPrivate.qll
View File

@@ -45,9 +45,9 @@ abstract class NodeImpl extends Node {
abstract DotNet::Type getTypeImpl();
/** Gets the type of this node used for type pruning. */
Gvn::GvnType getDataFlowType() {
DataFlowType getDataFlowType() {
forceCachingInSameStage() and
exists(Type t0 | result = Gvn::getGlobalValueNumber(t0) |
exists(Type t0 | result.asGvnType() = Gvn::getGlobalValueNumber(t0) |
t0 = getCSharpType(this.getType())
or
not exists(getCSharpType(this.getType())) and
@@ -533,8 +533,42 @@ module LocalFlow {
) and
not exists(getALastEvalNode(result))
}
/**
* Holds if the value of `node2` is given by `node1`.
*/
predicate localMustFlowStep(Node node1, Node node2) {
exists(Callable c, Expr e |
node1.(InstanceParameterNode).getCallable() = c and
node2.asExpr() = e and
(e instanceof ThisAccess or e instanceof BaseAccess) and
c = e.getEnclosingCallable()
)
or
hasNodePath(any(LocalExprStepConfiguration x), node1, node2) and
(
node2 instanceof SsaDefinitionExtNode or
node2.asExpr() instanceof Cast or
node2.asExpr() instanceof AssignExpr
)
or
exists(SsaImpl::Definition def |
def = getSsaDefinitionExt(node1) and
exists(SsaImpl::getAReadAtNode(def, node2.(ExprNode).getControlFlowNode()))
)
or
delegateCreationStep(node1, node2)
or
node1 =
unique(FlowSummaryNode n1 |
FlowSummaryImpl::Private::Steps::summaryLocalStep(n1.getSummaryNode(),
node2.(FlowSummaryNode).getSummaryNode(), true)
)
}
}
predicate localMustFlowStep = LocalFlow::localMustFlowStep/2;
/**
* This is the local flow predicate that is used as a building block in global
* data flow. It excludes SSA flow through instance fields, as flow through fields
@@ -761,16 +795,16 @@ private Type getCSharpType(DotNet::Type t) {
result.matchesHandle(t)
}
private class RelevantDataFlowType extends DataFlowType {
RelevantDataFlowType() { this = any(NodeImpl n).getDataFlowType() }
private class RelevantGvnType extends Gvn::GvnType {
RelevantGvnType() { this = any(NodeImpl n).getDataFlowType().asGvnType() }
}
/** A GVN type that is either a `DataFlowType` or unifiable with a `DataFlowType`. */
private class DataFlowTypeOrUnifiable extends Gvn::GvnType {
pragma[nomagic]
DataFlowTypeOrUnifiable() {
this instanceof RelevantDataFlowType or
Gvn::unifiable(any(RelevantDataFlowType t), this)
this instanceof RelevantGvnType or
Gvn::unifiable(any(RelevantGvnType t), this)
}
}
@@ -781,7 +815,7 @@ private TypeParameter getATypeParameterSubType(DataFlowTypeOrUnifiable t) {
}
pragma[noinline]
private TypeParameter getATypeParameterSubTypeRestricted(RelevantDataFlowType t) {
private TypeParameter getATypeParameterSubTypeRestricted(RelevantGvnType t) {
result = getATypeParameterSubType(t)
}
@@ -797,7 +831,7 @@ private Gvn::GvnType getANonTypeParameterSubType(DataFlowTypeOrUnifiable t) {
}
pragma[noinline]
private Gvn::GvnType getANonTypeParameterSubTypeRestricted(RelevantDataFlowType t) {
private Gvn::GvnType getANonTypeParameterSubTypeRestricted(RelevantGvnType t) {
result = getANonTypeParameterSubType(t)
}
@@ -834,6 +868,7 @@ private module Cached {
c = any(DataFlowCallable dfc).asCallable() and
not c.(Modifiable).isStatic()
} or
TDelegateSelfReferenceNode(Callable c) { lambdaCreationExpr(_, c) } or
TYieldReturnNode(ControlFlow::Nodes::ElementNode cfn) {
any(Callable c).canYieldReturn(cfn.getAstNode())
} or
@@ -917,7 +952,7 @@ private module Cached {
TSyntheticFieldApproxContent()
pragma[nomagic]
private predicate commonSubTypeGeneral(DataFlowTypeOrUnifiable t1, RelevantDataFlowType t2) {
private predicate commonSubTypeGeneral(DataFlowTypeOrUnifiable t1, RelevantGvnType t2) {
not t1 instanceof Gvn::TypeParameterGvnType and
t1 = t2
or
@@ -931,17 +966,20 @@ private module Cached {
* `t2` are allowed to be type parameters.
*/
cached
predicate commonSubType(RelevantDataFlowType t1, RelevantDataFlowType t2) {
commonSubTypeGeneral(t1, t2)
}
predicate commonSubType(RelevantGvnType t1, RelevantGvnType t2) { commonSubTypeGeneral(t1, t2) }
cached
predicate commonSubTypeUnifiableLeft(RelevantDataFlowType t1, RelevantDataFlowType t2) {
predicate commonSubTypeUnifiableLeft(RelevantGvnType t1, RelevantGvnType t2) {
exists(Gvn::GvnType t |
Gvn::unifiable(t1, t) and
commonSubTypeGeneral(t, t2)
)
}
cached
newtype TDataFlowType =
TGvnDataFlowType(Gvn::GvnType t) or
TDelegateDataFlowType(Callable lambda) { lambdaCreationExpr(_, lambda) }
}
import Cached
@@ -1087,6 +1125,37 @@ private module ParameterNodes {
override string toStringImpl() { result = "this" }
}
/**
* The value of a delegate itself at function entry, viewed as a node in a data
* flow graph.
*
* This is used for improving lambda dispatch, and will eventually also be
* used for tracking flow through captured variables.
*/
private class DelegateSelfReferenceNode extends ParameterNodeImpl, TDelegateSelfReferenceNode {
private Callable callable;
DelegateSelfReferenceNode() { this = TDelegateSelfReferenceNode(callable) }
final Callable getCallable() { result = callable }
override predicate isParameterOf(DataFlowCallable c, ParameterPosition pos) {
callable = c.asCallable() and pos.isDelegateSelf()
}
override ControlFlow::Node getControlFlowNodeImpl() { none() }
override DataFlowCallable getEnclosingCallableImpl() { result.asCallable() = callable }
override Location getLocationImpl() { result = callable.getLocation() }
override DotNet::Type getTypeImpl() { none() }
override DataFlowType getDataFlowType() { callable = result.asDelegate() }
override string toStringImpl() { result = "delegate self in " + callable }
}
/** An implicit entry definition for a captured variable. */
class SsaCapturedEntryDefinition extends Ssa::ImplicitEntryDefinition {
private LocalScopeVariable v;
@@ -1200,6 +1269,18 @@ private module ArgumentNodes {
}
}
/** A data-flow node that represents a delegate passed into itself. */
class DelegateSelfArgumentNode extends ArgumentNodeImpl {
private DataFlowCall call_;
DelegateSelfArgumentNode() { lambdaCallExpr(call_, this) }
override predicate argumentOf(DataFlowCall call, ArgumentPosition pos) {
call = call_ and
pos.isDelegateSelf()
}
}
/**
* The value of a captured variable as an implicit argument of a call, viewed
* as a node in a data flow graph.
@@ -1953,41 +2034,92 @@ predicate isUnreachableInCall(Node n, DataFlowCall call) {
* For example, `Func<T, int>` and `Func<S, int>` are mapped to the same
* `DataFlowType`, while `Func<T, int>` and `Func<string, int>` are not, because
* `string` is not a type parameter.
*
* For delegates, we use the delegate itself instead of its type, in order to
* improve dispatch.
*/
class DataFlowType = Gvn::GvnType;
class DataFlowType extends TDataFlowType {
Gvn::GvnType asGvnType() { this = TGvnDataFlowType(result) }
Callable asDelegate() { this = TDelegateDataFlowType(result) }
/**
* Gets an expression that creates a delegate of this type.
*
* For methods used as method groups in calls there can be multiple
* creations associated with the same type.
*/
Expr getADelegateCreation() {
exists(Callable callable |
lambdaCreationExpr(result, callable) and
this = TDelegateDataFlowType(callable)
)
}
final string toString() {
result = this.asGvnType().toString()
or
result = this.asDelegate().toString()
}
}
/** Gets the type of `n` used for type pruning. */
Gvn::GvnType getNodeType(Node n) { result = n.(NodeImpl).getDataFlowType() }
DataFlowType getNodeType(Node n) {
result = n.(NodeImpl).getDataFlowType() and
not lambdaCreation(n, _, _) and
not delegateCreationStep(_, n)
or
exists(Node arg |
delegateCreationStep(arg, n) and
result = getNodeType(arg)
)
or
n.asExpr() = result.getADelegateCreation()
}
/** Gets a string representation of a `DataFlowType`. */
string ppReprType(DataFlowType t) { result = t.toString() }
private class DataFlowNullType extends DataFlowType {
private class DataFlowNullType extends Gvn::GvnType {
DataFlowNullType() { this = Gvn::getGlobalValueNumber(any(NullType nt)) }
pragma[noinline]
predicate isConvertibleTo(DataFlowType t) {
predicate isConvertibleTo(Gvn::GvnType t) {
defaultNullConversion(_, any(Type t0 | t = Gvn::getGlobalValueNumber(t0)))
}
}
private class DataFlowUnknownType extends DataFlowType {
DataFlowUnknownType() { this = Gvn::getGlobalValueNumber(any(UnknownType ut)) }
}
private predicate uselessTypebound(DataFlowType t) {
t instanceof DataFlowUnknownType or
t instanceof Gvn::TypeParameterGvnType
private class GvnUnknownType extends Gvn::GvnType {
GvnUnknownType() { this = Gvn::getGlobalValueNumber(any(UnknownType ut)) }
}
pragma[nomagic]
predicate typeStrongerThan(DataFlowType t1, DataFlowType t2) {
t1 != t2 and
t1 = getANonTypeParameterSubTypeRestricted(t2)
or
t1 instanceof RelevantDataFlowType and
not uselessTypebound(t1) and
uselessTypebound(t2)
private predicate uselessTypebound(DataFlowType dt) {
dt.asGvnType() =
any(Gvn::GvnType t |
t instanceof GvnUnknownType or
t instanceof Gvn::TypeParameterGvnType
)
}
pragma[inline]
private predicate compatibleTypesDelegateLeft(DataFlowType dt1, DataFlowType dt2) {
exists(Gvn::GvnType t1, Gvn::GvnType t2 |
t1 = exprNode(dt1.getADelegateCreation()).(NodeImpl).getDataFlowType().asGvnType() and
t2 = dt2.asGvnType()
|
commonSubType(t1, t2)
or
commonSubTypeUnifiableLeft(t1, t2)
or
commonSubTypeUnifiableLeft(t2, t1)
or
t2.(DataFlowNullType).isConvertibleTo(t1)
or
t2 instanceof Gvn::TypeParameterGvnType
or
t2 instanceof GvnUnknownType
)
}
/**
@@ -1995,24 +2127,47 @@ predicate typeStrongerThan(DataFlowType t1, DataFlowType t2) {
* a node of type `t1` to a node of type `t2`.
*/
pragma[inline]
predicate compatibleTypes(DataFlowType t1, DataFlowType t2) {
commonSubType(t1, t2)
predicate compatibleTypes(DataFlowType dt1, DataFlowType dt2) {
exists(Gvn::GvnType t1, Gvn::GvnType t2 |
t1 = dt1.asGvnType() and
t2 = dt2.asGvnType()
|
commonSubType(t1, t2)
or
commonSubTypeUnifiableLeft(t1, t2)
or
commonSubTypeUnifiableLeft(t2, t1)
or
t1.(DataFlowNullType).isConvertibleTo(t2)
or
t2.(DataFlowNullType).isConvertibleTo(t1)
or
t1 instanceof Gvn::TypeParameterGvnType
or
t2 instanceof Gvn::TypeParameterGvnType
or
t1 instanceof GvnUnknownType
or
t2 instanceof GvnUnknownType
)
or
commonSubTypeUnifiableLeft(t1, t2)
compatibleTypesDelegateLeft(dt1, dt2)
or
commonSubTypeUnifiableLeft(t2, t1)
compatibleTypesDelegateLeft(dt2, dt1)
or
t1.(DataFlowNullType).isConvertibleTo(t2)
dt1.asDelegate() = dt2.asDelegate()
}
pragma[nomagic]
predicate typeStrongerThan(DataFlowType t1, DataFlowType t2) {
t1 != t2 and
t1.asGvnType() = getANonTypeParameterSubTypeRestricted(t2.asGvnType())
or
t2.(DataFlowNullType).isConvertibleTo(t1)
t1.asGvnType() instanceof RelevantGvnType and
not uselessTypebound(t1) and
uselessTypebound(t2)
or
t1 instanceof Gvn::TypeParameterGvnType
or
t2 instanceof Gvn::TypeParameterGvnType
or
t1 instanceof DataFlowUnknownType
or
t2 instanceof DataFlowUnknownType
compatibleTypesDelegateLeft(t1, t2)
}
/**
@@ -2184,17 +2339,20 @@ int accessPathLimit() { result = 5 }
*/
predicate forceHighPrecision(Content c) { c instanceof ElementContent }
private predicate lambdaCreationExpr(Expr creation, Callable c) {
c =
[
creation.(AnonymousFunctionExpr),
creation.(CallableAccess).getTarget().getUnboundDeclaration(),
creation.(AddressOfExpr).getOperand().(CallableAccess).getTarget().getUnboundDeclaration()
]
}
class LambdaCallKind = Unit;
/** Holds if `creation` is an expression that creates a delegate for `c`. */
predicate lambdaCreation(Node creation, LambdaCallKind kind, DataFlowCallable c) {
exists(Expr e | e = creation.asExpr() |
c.asCallable() =
[
e.(AnonymousFunctionExpr), e.(CallableAccess).getTarget().getUnboundDeclaration(),
e.(AddressOfExpr).getOperand().(CallableAccess).getTarget().getUnboundDeclaration()
]
) and
lambdaCreationExpr(creation.asExpr(), c.asCallable()) and
exists(kind)
}
@@ -2216,19 +2374,29 @@ private class LambdaConfiguration extends ControlFlowReachabilityConfiguration {
}
}
private predicate lambdaCallExpr(DataFlowCall call, ExprNode receiver) {
exists(LambdaConfiguration x, DelegateLikeCall dc |
x.hasExprPath(dc.getExpr(), receiver.getControlFlowNode(), dc, call.getControlFlowNode())
)
}
/** Holds if `call` is a lambda call where `receiver` is the lambda expression. */
predicate lambdaCall(DataFlowCall call, LambdaCallKind kind, Node receiver) {
(
exists(LambdaConfiguration x, DelegateLikeCall dc |
x.hasExprPath(dc.getExpr(), receiver.(ExprNode).getControlFlowNode(), dc,
call.getControlFlowNode())
)
lambdaCallExpr(call, receiver)
or
receiver.(FlowSummaryNode).getSummaryNode() = call.(SummaryCall).getReceiver()
) and
exists(kind)
}
private predicate delegateCreationStep(Node nodeFrom, Node nodeTo) {
exists(LambdaConfiguration x, DelegateCreation dc |
x.hasExprPath(dc.getArgument(), nodeFrom.(ExprNode).getControlFlowNode(), dc,
nodeTo.(ExprNode).getControlFlowNode())
)
}
/** Extra data-flow steps needed for lambda flow analysis. */
predicate additionalLambdaFlowStep(Node nodeFrom, Node nodeTo, boolean preservesValue) {
exists(SsaImpl::DefinitionExt def |
@@ -2237,11 +2405,8 @@ predicate additionalLambdaFlowStep(Node nodeFrom, Node nodeTo, boolean preserves
preservesValue = true
)
or
exists(LambdaConfiguration x, DelegateCreation dc |
x.hasExprPath(dc.getArgument(), nodeFrom.(ExprNode).getControlFlowNode(), dc,
nodeTo.(ExprNode).getControlFlowNode()) and
preservesValue = false
)
delegateCreationStep(nodeFrom, nodeTo) and
preservesValue = true
or
exists(AddEventExpr aee |
nodeFrom.asExpr() = aee.getRValue() and
@@ -2388,12 +2553,3 @@ module Csv {
)
}
}
/**
* Gets an additional term that is added to the `join` and `branch` computations to reflect
* an additional forward or backwards branching factor that is not taken into account
* when calculating the (virtual) dispatch cost.
*
* Argument `arg` is part of a path from a source to a sink, and `p` is the target parameter.
*/
int getAdditionalFlowIntoCallNodeTerm(ArgumentNode arg, ParameterNode p) { none() }

33
csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImplSpecific.qll
View File

@@ -35,14 +35,14 @@ private module SyntheticGlobals {
DataFlowCallable inject(SummarizedCallable c) { result.asSummarizedCallable() = c }
/** Gets the parameter position of the instance parameter. */
ArgumentPosition callbackSelfParameterPosition() { none() } // disables implicit summary flow to `this` for callbacks
ArgumentPosition callbackSelfParameterPosition() { result.isDelegateSelf() }
/** Gets the synthesized data-flow call for `receiver`. */
SummaryCall summaryDataFlowCall(SummaryNode receiver) { receiver = result.getReceiver() }
/** Gets the type of content `c`. */
DataFlowType getContentType(Content c) {
exists(Type t | result = Gvn::getGlobalValueNumber(t) |
exists(Type t | result.asGvnType() = Gvn::getGlobalValueNumber(t) |
t = c.(FieldContent).getField().getType()
or
t = c.(PropertyContent).getProperty().getType()
@@ -56,7 +56,7 @@ DataFlowType getContentType(Content c) {
/** Gets the type of the parameter at the given position. */
DataFlowType getParameterType(SummarizedCallable c, ParameterPosition pos) {
exists(Type t | result = Gvn::getGlobalValueNumber(t) |
exists(Type t | result.asGvnType() = Gvn::getGlobalValueNumber(t) |
exists(int i |
pos.getPosition() = i and
t = c.getParameter(i).getType()
@@ -69,7 +69,7 @@ DataFlowType getParameterType(SummarizedCallable c, ParameterPosition pos) {
/** Gets the return type of kind `rk` for callable `c`. */
DataFlowType getReturnType(DotNet::Callable c, ReturnKind rk) {
exists(Type t | result = Gvn::getGlobalValueNumber(t) |
exists(Type t | result.asGvnType() = Gvn::getGlobalValueNumber(t) |
rk instanceof NormalReturnKind and
(
t = c.(Constructor).getDeclaringType()
@@ -88,10 +88,13 @@ DataFlowType getReturnType(DotNet::Callable c, ReturnKind rk) {
*/
DataFlowType getCallbackParameterType(DataFlowType t, ArgumentPosition pos) {
exists(SystemLinqExpressions::DelegateExtType dt |
t = Gvn::getGlobalValueNumber(dt) and
result =
t.asGvnType() = Gvn::getGlobalValueNumber(dt) and
result.asGvnType() =
Gvn::getGlobalValueNumber(dt.getDelegateType().getParameter(pos.getPosition()).getType())
)
or
pos.isDelegateSelf() and
result = t
}
/**
@@ -101,15 +104,15 @@ DataFlowType getCallbackParameterType(DataFlowType t, ArgumentPosition pos) {
DataFlowType getCallbackReturnType(DataFlowType t, ReturnKind rk) {
rk instanceof NormalReturnKind and
exists(SystemLinqExpressions::DelegateExtType dt |
t = Gvn::getGlobalValueNumber(dt) and
result = Gvn::getGlobalValueNumber(dt.getDelegateType().getReturnType())
t.asGvnType() = Gvn::getGlobalValueNumber(dt) and
result.asGvnType() = Gvn::getGlobalValueNumber(dt.getDelegateType().getReturnType())
)
}
/** Gets the type of synthetic global `sg`. */
DataFlowType getSyntheticGlobalType(SummaryComponent::SyntheticGlobal sg) {
exists(sg) and
result = Gvn::getGlobalValueNumber(any(ObjectType t))
result.asGvnType() = Gvn::getGlobalValueNumber(any(ObjectType t))
}
/**
@@ -223,6 +226,9 @@ string getParameterPosition(ParameterPosition pos) {
or
pos.isThisParameter() and
result = "this"
or
pos.isDelegateSelf() and
result = "delegate-self"
}
/** Gets the textual representation of an argument position in the format used for flow summaries. */
@@ -231,6 +237,9 @@ string getArgumentPosition(ArgumentPosition pos) {
or
pos.isQualifier() and
result = "this"
or
pos.isDelegateSelf() and
result = "delegate-self"
}
/** Holds if input specification component `c` needs a reference. */
@@ -312,6 +321,9 @@ ArgumentPosition parseParamBody(string s) {
or
s = "this" and
result.isQualifier()
or
s = "delegate-self" and
result.isDelegateSelf()
}
/** Gets the parameter position obtained by parsing `X` in `Argument[X]`. */
@@ -321,4 +333,7 @@ ParameterPosition parseArgBody(string s) {
or
s = "this" and
result.isThisParameter()
or
s = "delegate-self" and
result.isDelegateSelf()
}

107
csharp/ql/lib/semmle/code/csharp/security/auth/ActionMethods.qll Normal file
View File

@@ -0,0 +1,107 @@
/** Common definitions for queries checking for access control measures on action methods. */
import csharp
import semmle.code.csharp.frameworks.microsoft.AspNetCore
import semmle.code.csharp.frameworks.system.web.UI
/** A method representing an action for a web endpoint. */
abstract class ActionMethod extends Method {
/**
* Gets a string that can indicate what this method does to determine if it should have an auth check;
* such as its method name, class name, or file path.
*/
string getADescription() {
result = [this.getName(), this.getDeclaringType().getBaseClass*().getName(), this.getARoute()]
}
/** Holds if this method may represent a stateful action such as editing or deleting */
predicate isEdit() {
exists(string str |
str =
this.getADescription()
// separate camelCase words
.regexpReplaceAll("([a-z])([A-Z])", "$1_$2") and
str.regexpMatch("(?i).*(edit|delete|modify|change).*") and
not str.regexpMatch("(?i).*(on_?change|changed).*")
)
}
/** Holds if this method may be intended to be restricted to admin users */
predicate isAdmin() {
this.getADescription()
// separate camelCase words
.regexpReplaceAll("([a-z])([A-Z])", "$1_$2")
.regexpMatch("(?i).*(admin|superuser).*")
}
/** Gets a callable for which if it contains an auth check, this method should be considered authenticated. */
Callable getAnAuthorizingCallable() { result = this }
/**
* Gets a possible url route that could refer to this action,
* which would be covered by `<location>` configurations specifying a prefix of it.
*/
string getARoute() { result = this.getDeclaringType().getFile().getRelativePath() }
}
/** An action method in the MVC framework. */
private class MvcActionMethod extends ActionMethod {
MvcActionMethod() { this = any(MicrosoftAspNetCoreMvcController c).getAnActionMethod() }
}
/** An action method on a subclass of `System.Web.UI.Page`. */
private class WebFormActionMethod extends ActionMethod {
WebFormActionMethod() {
this.getDeclaringType().getBaseClass+() instanceof SystemWebUIPageClass and
this.getAParameter().getType().getName().matches("%EventArgs")
}
override Callable getAnAuthorizingCallable() {
result = super.getAnAuthorizingCallable()
or
pageLoad(result, this.getDeclaringType())
}
override string getARoute() {
exists(string physicalRoute | physicalRoute = super.getARoute() |
result = physicalRoute
or
exists(string absolutePhysical |
virtualRouteMapping(result, absolutePhysical) and
physicalRouteMatches(absolutePhysical, physicalRoute)
)
)
}
}
pragma[nomagic]
private predicate pageLoad(Callable c, Type decl) {
c.getName() = "Page_Load" and
decl = c.getDeclaringType()
}
/**
* Holds if `virtualRoute` is a URL path
* that can map to the corresponding `physicalRoute` filepath
* through a call to `MapPageRoute`
*/
private predicate virtualRouteMapping(string virtualRoute, string physicalRoute) {
exists(MethodCall mapPageRouteCall, StringLiteral virtualLit, StringLiteral physicalLit |
mapPageRouteCall
.getTarget()
.hasQualifiedName("System.Web.Routing", "RouteCollection", "MapPageRoute") and
virtualLit = mapPageRouteCall.getArgument(1) and
physicalLit = mapPageRouteCall.getArgument(2) and
virtualLit.getValue() = virtualRoute and
physicalLit.getValue() = physicalRoute
)
}
/** Holds if the filepath `route` can refer to `actual` after expanding a '~". */
bindingset[route, actual]
private predicate physicalRouteMatches(string route, string actual) {
route = actual
or
route.charAt(0) = "~" and
exists(string dir | actual = dir + route.suffix(1) + ".cs")
}

Some files were not shown because too many files have changed in this diff Show More