From da8cc13506f99b310ea6505b1ee319bb9b22dde8 Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Mon, 11 Mar 2024 09:57:02 +0000 Subject: [PATCH 01/97] go extractor: avoid long string concatenations When we see "a" + "b" + "c" + "d", do not add a row to the constvalues table for the intermiediate strings "ab" and "abc". We still have entries for the string literals ("a", "b", "c", and "d") and the whole string concatenation ("abcd"). --- go/extractor/extractor.go | 142 ++++++++++++++++++++++---------------- 1 file changed, 83 insertions(+), 59 deletions(-) diff --git a/go/extractor/extractor.go b/go/extractor/extractor.go index f2ba68a20f0..97f3ed743fb 100644 --- a/go/extractor/extractor.go +++ b/go/extractor/extractor.go @@ -794,7 +794,7 @@ func extractLocalScope(tw *trap.Writer, scope *types.Scope, parentScopeLabel tra func extractFileNode(tw *trap.Writer, nd *ast.File) { lbl := tw.Labeler.FileLabel() - extractExpr(tw, nd.Name, lbl, 0) + extractExpr(tw, nd.Name, lbl, 0, false) for i, decl := range nd.Decls { extractDecl(tw, decl, lbl, i) @@ -851,7 +851,7 @@ func emitScopeNodeInfo(tw *trap.Writer, nd ast.Node, lbl trap.Label) { } // extractExpr extracts AST information for the given expression and all its subexpressions -func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { +func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int, skipExtractingValue bool) { if expr == nil { return } @@ -900,7 +900,7 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { return } kind = dbscheme.EllipsisExpr.Index() - extractExpr(tw, expr.Elt, lbl, 0) + extractExpr(tw, expr.Elt, lbl, 0, false) case *ast.BasicLit: if expr == nil { return @@ -932,28 +932,28 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { return } kind = dbscheme.FuncLitExpr.Index() - extractExpr(tw, expr.Type, lbl, 0) + extractExpr(tw, expr.Type, lbl, 0, false) extractStmt(tw, expr.Body, lbl, 1) case *ast.CompositeLit: if expr == nil { return } kind = dbscheme.CompositeLitExpr.Index() - extractExpr(tw, expr.Type, lbl, 0) + extractExpr(tw, expr.Type, lbl, 0, false) extractExprs(tw, expr.Elts, lbl, 1, 1) case *ast.ParenExpr: if expr == nil { return } kind = dbscheme.ParenExpr.Index() - extractExpr(tw, expr.X, lbl, 0) + extractExpr(tw, expr.X, lbl, 0, false) case *ast.SelectorExpr: if expr == nil { return } kind = dbscheme.SelectorExpr.Index() - extractExpr(tw, expr.X, lbl, 0) - extractExpr(tw, expr.Sel, lbl, 1) + extractExpr(tw, expr.X, lbl, 0, false) + extractExpr(tw, expr.Sel, lbl, 1, false) case *ast.IndexExpr: if expr == nil { return @@ -974,8 +974,8 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { kind = dbscheme.IndexExpr.Index() } } - extractExpr(tw, expr.X, lbl, 0) - extractExpr(tw, expr.Index, lbl, 1) + extractExpr(tw, expr.X, lbl, 0, false) + extractExpr(tw, expr.Index, lbl, 1, false) case *ast.IndexListExpr: if expr == nil { return @@ -993,30 +993,30 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { kind = dbscheme.GenericTypeInstantiationExpr.Index() } } - extractExpr(tw, expr.X, lbl, 0) + extractExpr(tw, expr.X, lbl, 0, false) extractExprs(tw, expr.Indices, lbl, 1, 1) case *ast.SliceExpr: if expr == nil { return } kind = dbscheme.SliceExpr.Index() - extractExpr(tw, expr.X, lbl, 0) - extractExpr(tw, expr.Low, lbl, 1) - extractExpr(tw, expr.High, lbl, 2) - extractExpr(tw, expr.Max, lbl, 3) + extractExpr(tw, expr.X, lbl, 0, false) + extractExpr(tw, expr.Low, lbl, 1, false) + extractExpr(tw, expr.High, lbl, 2, false) + extractExpr(tw, expr.Max, lbl, 3, false) case *ast.TypeAssertExpr: if expr == nil { return } kind = dbscheme.TypeAssertExpr.Index() - extractExpr(tw, expr.X, lbl, 0) - extractExpr(tw, expr.Type, lbl, 1) + extractExpr(tw, expr.X, lbl, 0, false) + extractExpr(tw, expr.Type, lbl, 1, false) case *ast.CallExpr: if expr == nil { return } kind = dbscheme.CallOrConversionExpr.Index() - extractExpr(tw, expr.Fun, lbl, 0) + extractExpr(tw, expr.Fun, lbl, 0, false) extractExprs(tw, expr.Args, lbl, 1, 1) if expr.Ellipsis.IsValid() { dbscheme.HasEllipsisTable.Emit(tw, lbl) @@ -1026,14 +1026,14 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { return } kind = dbscheme.StarExpr.Index() - extractExpr(tw, expr.X, lbl, 0) + extractExpr(tw, expr.X, lbl, 0, false) case *ast.KeyValueExpr: if expr == nil { return } kind = dbscheme.KeyValueExpr.Index() - extractExpr(tw, expr.Key, lbl, 0) - extractExpr(tw, expr.Value, lbl, 1) + extractExpr(tw, expr.Key, lbl, 0, false) + extractExpr(tw, expr.Value, lbl, 1, false) case *ast.UnaryExpr: if expr == nil { return @@ -1047,7 +1047,7 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { } kind = tp.Index() } - extractExpr(tw, expr.X, lbl, 0) + extractExpr(tw, expr.X, lbl, 0, false) case *ast.BinaryExpr: if expr == nil { return @@ -1062,16 +1062,17 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { log.Fatalf("unsupported binary operator %s", expr.Op) } kind = tp.Index() - extractExpr(tw, expr.X, lbl, 0) - extractExpr(tw, expr.Y, lbl, 1) + skipLeft := skipExtractingValueForLeftOperand(tw, expr) + extractExpr(tw, expr.X, lbl, 0, skipLeft) + extractExpr(tw, expr.Y, lbl, 1, false) } case *ast.ArrayType: if expr == nil { return } kind = dbscheme.ArrayTypeExpr.Index() - extractExpr(tw, expr.Len, lbl, 0) - extractExpr(tw, expr.Elt, lbl, 1) + extractExpr(tw, expr.Len, lbl, 0, false) + extractExpr(tw, expr.Elt, lbl, 1, false) case *ast.StructType: if expr == nil { return @@ -1100,8 +1101,8 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { return } kind = dbscheme.MapTypeExpr.Index() - extractExpr(tw, expr.Key, lbl, 0) - extractExpr(tw, expr.Value, lbl, 1) + extractExpr(tw, expr.Key, lbl, 0, false) + extractExpr(tw, expr.Value, lbl, 1, false) case *ast.ChanType: if expr == nil { return @@ -1111,13 +1112,15 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { log.Fatalf("unsupported channel direction %v", expr.Dir) } kind = tp.Index() - extractExpr(tw, expr.Value, lbl, 0) + extractExpr(tw, expr.Value, lbl, 0, false) default: log.Fatalf("unknown expression of type %T", expr) } dbscheme.ExprsTable.Emit(tw, lbl, kind, parent, idx) extractNodeLocation(tw, expr, lbl) - extractValueOf(tw, expr, lbl) + if !skipExtractingValue { + extractValueOf(tw, expr, lbl) + } } // extractExprs extracts AST information for a list of expressions, which are children of @@ -1128,7 +1131,7 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { func extractExprs(tw *trap.Writer, exprs []ast.Expr, parent trap.Label, idx int, dir int) { if exprs != nil { for _, expr := range exprs { - extractExpr(tw, expr, parent, idx) + extractExpr(tw, expr, parent, idx, false) idx += dir } } @@ -1194,11 +1197,11 @@ func extractFields(tw *trap.Writer, fields *ast.FieldList, parent trap.Label, id extractNodeLocation(tw, field, lbl) if field.Names != nil { for i, name := range field.Names { - extractExpr(tw, name, lbl, i+1) + extractExpr(tw, name, lbl, i+1, false) } } - extractExpr(tw, field.Type, lbl, 0) - extractExpr(tw, field.Tag, lbl, -1) + extractExpr(tw, field.Type, lbl, 0, false) + extractExpr(tw, field.Tag, lbl, -1, false) extractDoc(tw, field.Doc, lbl) idx += dir } @@ -1229,21 +1232,21 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { return } kind = dbscheme.LabeledStmtType.Index() - extractExpr(tw, stmt.Label, lbl, 0) + extractExpr(tw, stmt.Label, lbl, 0, false) extractStmt(tw, stmt.Stmt, lbl, 1) case *ast.ExprStmt: if stmt == nil { return } kind = dbscheme.ExprStmtType.Index() - extractExpr(tw, stmt.X, lbl, 0) + extractExpr(tw, stmt.X, lbl, 0, false) case *ast.SendStmt: if stmt == nil { return } kind = dbscheme.SendStmtType.Index() - extractExpr(tw, stmt.Chan, lbl, 0) - extractExpr(tw, stmt.Value, lbl, 1) + extractExpr(tw, stmt.Chan, lbl, 0, false) + extractExpr(tw, stmt.Value, lbl, 1, false) case *ast.IncDecStmt: if stmt == nil { return @@ -1255,7 +1258,7 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { } else { log.Fatalf("unsupported increment/decrement operator %v", stmt.Tok) } - extractExpr(tw, stmt.X, lbl, 0) + extractExpr(tw, stmt.X, lbl, 0, false) case *ast.AssignStmt: if stmt == nil { return @@ -1272,13 +1275,13 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { return } kind = dbscheme.GoStmtType.Index() - extractExpr(tw, stmt.Call, lbl, 0) + extractExpr(tw, stmt.Call, lbl, 0, false) case *ast.DeferStmt: if stmt == nil { return } kind = dbscheme.DeferStmtType.Index() - extractExpr(tw, stmt.Call, lbl, 0) + extractExpr(tw, stmt.Call, lbl, 0, false) case *ast.ReturnStmt: kind = dbscheme.ReturnStmtType.Index() extractExprs(tw, stmt.Results, lbl, 0, 1) @@ -1298,7 +1301,7 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { default: log.Fatalf("unsupported branch statement type %v", stmt.Tok) } - extractExpr(tw, stmt.Label, lbl, 0) + extractExpr(tw, stmt.Label, lbl, 0, false) case *ast.BlockStmt: if stmt == nil { return @@ -1312,7 +1315,7 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { } kind = dbscheme.IfStmtType.Index() extractStmt(tw, stmt.Init, lbl, 0) - extractExpr(tw, stmt.Cond, lbl, 1) + extractExpr(tw, stmt.Cond, lbl, 1, false) extractStmt(tw, stmt.Body, lbl, 2) extractStmt(tw, stmt.Else, lbl, 3) emitScopeNodeInfo(tw, stmt, lbl) @@ -1330,7 +1333,7 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { } kind = dbscheme.ExprSwitchStmtType.Index() extractStmt(tw, stmt.Init, lbl, 0) - extractExpr(tw, stmt.Tag, lbl, 1) + extractExpr(tw, stmt.Tag, lbl, 1, false) extractStmt(tw, stmt.Body, lbl, 2) emitScopeNodeInfo(tw, stmt, lbl) case *ast.TypeSwitchStmt: @@ -1359,7 +1362,7 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { } kind = dbscheme.ForStmtType.Index() extractStmt(tw, stmt.Init, lbl, 0) - extractExpr(tw, stmt.Cond, lbl, 1) + extractExpr(tw, stmt.Cond, lbl, 1, false) extractStmt(tw, stmt.Post, lbl, 2) extractStmt(tw, stmt.Body, lbl, 3) emitScopeNodeInfo(tw, stmt, lbl) @@ -1368,9 +1371,9 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { return } kind = dbscheme.RangeStmtType.Index() - extractExpr(tw, stmt.Key, lbl, 0) - extractExpr(tw, stmt.Value, lbl, 1) - extractExpr(tw, stmt.X, lbl, 2) + extractExpr(tw, stmt.Key, lbl, 0, false) + extractExpr(tw, stmt.Value, lbl, 1, false) + extractExpr(tw, stmt.X, lbl, 2, false) extractStmt(tw, stmt.Body, lbl, 3) emitScopeNodeInfo(tw, stmt, lbl) default: @@ -1428,8 +1431,8 @@ func extractDecl(tw *trap.Writer, decl ast.Decl, parent trap.Label, idx int) { } kind = dbscheme.FuncDeclType.Index() extractFields(tw, decl.Recv, lbl, -1, -1) - extractExpr(tw, decl.Name, lbl, 0) - extractExpr(tw, decl.Type, lbl, 1) + extractExpr(tw, decl.Name, lbl, 0, false) + extractExpr(tw, decl.Type, lbl, 1, false) extractStmt(tw, decl.Body, lbl, 2) extractDoc(tw, decl.Doc, lbl) extractTypeParamDecls(tw, decl.Type.TypeParams, lbl) @@ -1455,8 +1458,8 @@ func extractSpec(tw *trap.Writer, spec ast.Spec, parent trap.Label, idx int) { return } kind = dbscheme.ImportSpecType.Index() - extractExpr(tw, spec.Name, lbl, 0) - extractExpr(tw, spec.Path, lbl, 1) + extractExpr(tw, spec.Name, lbl, 0, false) + extractExpr(tw, spec.Path, lbl, 1, false) extractDoc(tw, spec.Doc, lbl) case *ast.ValueSpec: if spec == nil { @@ -1464,9 +1467,9 @@ func extractSpec(tw *trap.Writer, spec ast.Spec, parent trap.Label, idx int) { } kind = dbscheme.ValueSpecType.Index() for i, name := range spec.Names { - extractExpr(tw, name, lbl, -(1 + i)) + extractExpr(tw, name, lbl, -(1 + i), false) } - extractExpr(tw, spec.Type, lbl, 0) + extractExpr(tw, spec.Type, lbl, 0, false) extractExprs(tw, spec.Values, lbl, 1, 1) extractDoc(tw, spec.Doc, lbl) case *ast.TypeSpec: @@ -1478,9 +1481,9 @@ func extractSpec(tw *trap.Writer, spec ast.Spec, parent trap.Label, idx int) { } else { kind = dbscheme.TypeDefSpecType.Index() } - extractExpr(tw, spec.Name, lbl, 0) + extractExpr(tw, spec.Name, lbl, 0, false) extractTypeParamDecls(tw, spec.TypeParams, lbl) - extractExpr(tw, spec.Type, lbl, 1) + extractExpr(tw, spec.Type, lbl, 1, false) extractDoc(tw, spec.Doc, lbl) } dbscheme.SpecsTable.Emit(tw, lbl, kind, parent, idx) @@ -1909,7 +1912,7 @@ func flattenBinaryExprTree(tw *trap.Writer, e ast.Expr, parent trap.Label, idx i idx = flattenBinaryExprTree(tw, binaryexpr.X, parent, idx) idx = flattenBinaryExprTree(tw, binaryexpr.Y, parent, idx) } else { - extractExpr(tw, e, parent, idx) + extractExpr(tw, e, parent, idx, false) idx = idx + 1 } return idx @@ -1931,10 +1934,10 @@ func extractTypeParamDecls(tw *trap.Writer, fields *ast.FieldList, parent trap.L extractNodeLocation(tw, field, lbl) if field.Names != nil { for i, name := range field.Names { - extractExpr(tw, name, lbl, i+1) + extractExpr(tw, name, lbl, i+1, false) } } - extractExpr(tw, field.Type, lbl, 0) + extractExpr(tw, field.Type, lbl, 0, false) extractDoc(tw, field.Doc, lbl) idx += 1 } @@ -2023,3 +2026,24 @@ func setTypeParamParent(tp *types.TypeParam, newobj types.Object) { log.Fatalf("Parent of type parameter '%s %s' being set to a different value: '%s' vs '%s'", tp.String(), tp.Constraint().String(), obj, newobj) } } + +// skipExtractingValueForLeftOperand returns true if the left operand of `be` +// should not have its value extracted because it is an intermediate value in a +// string concatenation - specifically that the right operand is a string +// literal +func skipExtractingValueForLeftOperand(tw *trap.Writer, be *ast.BinaryExpr) bool { + // check `be` has string type + tpVal := tw.Package.TypesInfo.Types[be] + if tpVal.Value == nil || tpVal.Value.Kind() != constant.String { + return false + } + // check that the right operand of `be` is a basic literal + if _, isBasicLit := be.Y.(*ast.BasicLit); !isBasicLit { + return false + } + // check that the left operand of `be` is not a basic literal + if _, isBasicLit := be.X.(*ast.BasicLit); isBasicLit { + return false + } + return true +} From 530c76ca8b49cf4338cc72124e3b49af28376fd2 Mon Sep 17 00:00:00 2001 From: Kevin Stubbings Date: Wed, 14 Dec 2022 20:59:47 -0800 Subject: [PATCH 02/97] Add New Sanitizers and Modify Old Ones --- .../go/security/TaintedPathCustomizations.qll | 46 ++++++++++++++++++- .../Security/CWE-022/TaintedPath.expected | 22 +++++++-- .../Security/CWE-022/TaintedPath.go | 29 +++++++++++- 3 files changed, 91 insertions(+), 6 deletions(-) diff --git a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll index 42edd470da2..3c0288493f5 100644 --- a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll +++ b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll @@ -74,19 +74,46 @@ module TaintedPath { } /** - * A call to `[file]path.Clean("/" + e)`, considered to sanitize `e` against path traversal. + * A call to `filepath.Clean("/" + e)`, considered to sanitize `e` against path traversal. */ class FilepathCleanSanitizer extends Sanitizer { FilepathCleanSanitizer() { exists(DataFlow::CallNode cleanCall, StringOps::Concatenation concatNode | cleanCall = - any(Function f | f.hasQualifiedName(["path", "path/filepath"], "Clean")).getACall() and + any(Function f | f.hasQualifiedName("path/filepath", "Clean")).getACall() and concatNode = cleanCall.getArgument(0) and concatNode.getOperand(0).asExpr().(StringLit).getValue() = "/" and this = cleanCall.getResult() ) } } + /** + * A call to `filepath.Base(e)`, considered to sanitize `e` against path traversal. + */ + class FilepathBaseSanitizer extends Sanitizer { + FilepathBaseSanitizer() { + exists(Function f, FunctionOutput outp | + f.hasQualifiedName("path/filepath", "Base") and + outp.isResult(0) and + this = outp.getNode(f.getACall()) + ) + } + } + + /** An call to ParseMultipartForm creates multipart.Form and cleans mutlpart.Form.FileHeader.Filename using path.Base() */ + class MultipartClean extends Sanitizer { + MultipartClean() { + exists(DataFlow::FieldReadNode frn, ControlFlow::Node node, DataFlow::CallNode cleanCall, Method get | + get.hasQualifiedName("net/http","Request", "ParseMultipartForm") and + cleanCall = get.getACall() and + cleanCall.asInstruction() = node and + frn.getField().hasQualifiedName("mime/multipart", "FileHeader", "Filename") and + node.getASuccessor*() = frn.asInstruction() + | + this = frn.getBase() + ) + } + } /** * A check of the form `!strings.Contains(nd, "..")`, considered as a sanitizer guard for @@ -105,6 +132,21 @@ module TaintedPath { branch = false } } +/** + * A replacement of the form `!strings.ReplaceAll(nd, "..")` or `!strings.ReplaceAll(nd, ".")`, considered as a sanitizer guard for + * path traversal. + */ + class DotDotReplace extends Sanitizer { + DotDotReplace() { + exists(DataFlow::CallNode cleanCall, DataFlow::Node valueNode | + cleanCall = + any(Function f | f.hasQualifiedName("strings", "ReplaceAll")).getACall() and + valueNode = cleanCall.getArgument(1) and + (valueNode.asExpr().(StringLit).getValue() = ".." or valueNode.asExpr().(StringLit).getValue() = ".") and + this = cleanCall.getResult() + ) + } + } /** * A node `nd` guarded by a check that ensures it is contained within some root folder, diff --git a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected index 34d4180a849..38d968a9e3b 100644 --- a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected +++ b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected @@ -1,19 +1,35 @@ edges +<<<<<<< HEAD | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:13:18:13:30 | call to Query | provenance | | | TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:16:29:16:40 | tainted_path | provenance | | | TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:20:57:20:68 | tainted_path | provenance | | | TaintedPath.go:20:57:20:68 | tainted_path | TaintedPath.go:20:28:20:69 | call to Join | provenance | | | tst.go:14:2:14:39 | ... := ...[1] | tst.go:17:41:17:56 | selection of Filename | provenance | | +======= +| TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:16:29:16:40 | tainted_path | +| TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:20:28:20:69 | call to Join | +| TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:67:28:67:57 | call to Clean | +| TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:77:28:77:56 | call to Base | +| tst.go:14:2:14:39 | ... := ...[1] : pointer type | tst.go:17:41:17:56 | selection of Filename | +>>>>>>> a45343fb6c (Add New Sanitizers and Modify Old Ones) nodes | TaintedPath.go:13:18:13:22 | selection of URL | semmle.label | selection of URL | | TaintedPath.go:13:18:13:30 | call to Query | semmle.label | call to Query | | TaintedPath.go:16:29:16:40 | tainted_path | semmle.label | tainted_path | | TaintedPath.go:20:28:20:69 | call to Join | semmle.label | call to Join | +<<<<<<< HEAD | TaintedPath.go:20:57:20:68 | tainted_path | semmle.label | tainted_path | | tst.go:14:2:14:39 | ... := ...[1] | semmle.label | ... := ...[1] | +======= +| TaintedPath.go:67:28:67:57 | call to Clean | semmle.label | call to Clean | +| TaintedPath.go:77:28:77:56 | call to Base | semmle.label | call to Base | +| tst.go:14:2:14:39 | ... := ...[1] : pointer type | semmle.label | ... := ...[1] : pointer type | +>>>>>>> a45343fb6c (Add New Sanitizers and Modify Old Ones) | tst.go:17:41:17:56 | selection of Filename | semmle.label | selection of Filename | subpaths #select -| TaintedPath.go:16:29:16:40 | tainted_path | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:16:29:16:40 | tainted_path | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | -| TaintedPath.go:20:28:20:69 | call to Join | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:20:28:20:69 | call to Join | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | -| tst.go:17:41:17:56 | selection of Filename | tst.go:14:2:14:39 | ... := ...[1] | tst.go:17:41:17:56 | selection of Filename | This path depends on a $@. | tst.go:14:2:14:39 | ... := ...[1] | user-provided value | +| TaintedPath.go:16:29:16:40 | tainted_path | TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:16:29:16:40 | tainted_path | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | +| TaintedPath.go:20:28:20:69 | call to Join | TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:20:28:20:69 | call to Join | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | +| TaintedPath.go:67:28:67:57 | call to Clean | TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:67:28:67:57 | call to Clean | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | +| TaintedPath.go:77:28:77:56 | call to Base | TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:77:28:77:56 | call to Base | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | +| tst.go:17:41:17:56 | selection of Filename | tst.go:14:2:14:39 | ... := ...[1] : pointer type | tst.go:17:41:17:56 | selection of Filename | This path depends on a $@. | tst.go:14:2:14:39 | ... := ...[1] | user-provided value | diff --git a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go index e663fc3b11d..629b7556e9b 100644 --- a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go +++ b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go @@ -31,6 +31,10 @@ func handler(w http.ResponseWriter, r *http.Request) { w.Write(data) } + // GOOD: Sanitized by strings.ReplaceAll and replaces all .. with empty string + data, _ = ioutil.ReadFile(strings.ReplaceAll(tainted_path, "..", "")) + w.Write(data) + // GOOD: This can only read inside the provided safe path _, err := filepath.Rel("/home/user/safepath", tainted_path) if err == nil { @@ -53,10 +57,33 @@ func handler(w http.ResponseWriter, r *http.Request) { w.Write(data) } - // GOOD: Sanitized by [file]path.Clean with a prepended '/' forcing interpretation + // GOOD: Sanitized by filepath.Clean with a prepended '/' forcing interpretation // as an absolute path, so that Clean will throw away any leading `..` components. data, _ = ioutil.ReadFile(filepath.Clean("/" + tainted_path)) w.Write(data) + + // BAD: Sanitized by path.Clean with a prepended '/' forcing interpretation + // as an absolute path, however is not sufficient for Windows paths. data, _ = ioutil.ReadFile(path.Clean("/" + tainted_path)) w.Write(data) + + // GOOD: Sanitized by filepath.Base with a prepended '/' forcing interpretation + // as an absolute path, so that Base will throw away any leading `..` components. + data, _ = ioutil.ReadFile(filepath.Base("/" + tainted_path)) + w.Write(data) + + // BAD: Sanitized by path.Base with a prepended '/' forcing interpretation + // as an absolute path, however is not sufficient for Windows paths. + data, _ = ioutil.ReadFile(path.Base("/" + tainted_path)) + w.Write(data) + + // GOOD: Multipart.Form.FileHeader.Filename sanitized by filepath.Base when calling ParseMultipartForm + r.ParseMultipartForm(32 << 20) + form := r.MultipartForm + files, ok := form.File["files"] + if !ok { + return + } + data, _ = ioutil.ReadFile(files[0].Filename) + w.Write(data) } From c9b49d3760db97e22ce266e73065c1c1d8b35879 Mon Sep 17 00:00:00 2001 From: Kevin Stubbings Date: Sun, 8 Oct 2023 16:24:25 -0700 Subject: [PATCH 03/97] resolve feedback --- .../go/security/TaintedPathCustomizations.qll | 17 ++++------- .../Security/CWE-022/TaintedPath.expected | 28 +++++++++++-------- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll index 3c0288493f5..815a628fec3 100644 --- a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll +++ b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll @@ -5,7 +5,7 @@ import go import semmle.go.dataflow.barrierguardutil.RegexpCheck - +import DataFlow /** * Provides extension points for customizing the taint tracking configuration for reasoning about * path-traversal vulnerabilities. @@ -100,17 +100,12 @@ module TaintedPath { } } - /** An call to ParseMultipartForm creates multipart.Form and cleans mutlpart.Form.FileHeader.Filename using path.Base() */ + /**An call to ParseMultipartForm creates multipart.Form and cleans mutlpart.Form.FileHeader.Filename using path.Base() */ class MultipartClean extends Sanitizer { MultipartClean() { - exists(DataFlow::FieldReadNode frn, ControlFlow::Node node, DataFlow::CallNode cleanCall, Method get | - get.hasQualifiedName("net/http","Request", "ParseMultipartForm") and - cleanCall = get.getACall() and - cleanCall.asInstruction() = node and + exists(DataFlow::FieldReadNode frn | frn.getField().hasQualifiedName("mime/multipart", "FileHeader", "Filename") and - node.getASuccessor*() = frn.asInstruction() - | - this = frn.getBase() + this = frn ) } } @@ -133,7 +128,7 @@ module TaintedPath { } } /** - * A replacement of the form `!strings.ReplaceAll(nd, "..")` or `!strings.ReplaceAll(nd, ".")`, considered as a sanitizer guard for + * A replacement of the form `!strings.ReplaceAll(nd, "..")` or `!strings.ReplaceAll(nd, ".")`, considered as a sanitizer for * path traversal. */ class DotDotReplace extends Sanitizer { @@ -142,7 +137,7 @@ module TaintedPath { cleanCall = any(Function f | f.hasQualifiedName("strings", "ReplaceAll")).getACall() and valueNode = cleanCall.getArgument(1) and - (valueNode.asExpr().(StringLit).getValue() = ".." or valueNode.asExpr().(StringLit).getValue() = ".") and + valueNode.asExpr().(StringLit).getValue() = ["..", "."] and this = cleanCall.getResult() ) } diff --git a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected index 38d968a9e3b..c3a4fc5a616 100644 --- a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected +++ b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected @@ -1,5 +1,6 @@ edges <<<<<<< HEAD +<<<<<<< HEAD | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:13:18:13:30 | call to Query | provenance | | | TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:16:29:16:40 | tainted_path | provenance | | | TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:20:57:20:68 | tainted_path | provenance | | @@ -12,24 +13,29 @@ edges | TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:77:28:77:56 | call to Base | | tst.go:14:2:14:39 | ... := ...[1] : pointer type | tst.go:17:41:17:56 | selection of Filename | >>>>>>> a45343fb6c (Add New Sanitizers and Modify Old Ones) +======= +| TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:13:18:13:30 | call to Query | +| TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:16:29:16:40 | tainted_path | +| TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:20:57:20:68 | tainted_path | +| TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:67:39:67:56 | ...+... | +| TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:77:38:77:55 | ...+... | +| TaintedPath.go:20:57:20:68 | tainted_path | TaintedPath.go:20:28:20:69 | call to Join | +| TaintedPath.go:67:39:67:56 | ...+... | TaintedPath.go:67:28:67:57 | call to Clean | +| TaintedPath.go:77:38:77:55 | ...+... | TaintedPath.go:77:28:77:56 | call to Base | +>>>>>>> db14838a4f (resolve feedback) nodes | TaintedPath.go:13:18:13:22 | selection of URL | semmle.label | selection of URL | | TaintedPath.go:13:18:13:30 | call to Query | semmle.label | call to Query | | TaintedPath.go:16:29:16:40 | tainted_path | semmle.label | tainted_path | | TaintedPath.go:20:28:20:69 | call to Join | semmle.label | call to Join | -<<<<<<< HEAD | TaintedPath.go:20:57:20:68 | tainted_path | semmle.label | tainted_path | -| tst.go:14:2:14:39 | ... := ...[1] | semmle.label | ... := ...[1] | -======= | TaintedPath.go:67:28:67:57 | call to Clean | semmle.label | call to Clean | +| TaintedPath.go:67:39:67:56 | ...+... | semmle.label | ...+... | | TaintedPath.go:77:28:77:56 | call to Base | semmle.label | call to Base | -| tst.go:14:2:14:39 | ... := ...[1] : pointer type | semmle.label | ... := ...[1] : pointer type | ->>>>>>> a45343fb6c (Add New Sanitizers and Modify Old Ones) -| tst.go:17:41:17:56 | selection of Filename | semmle.label | selection of Filename | +| TaintedPath.go:77:38:77:55 | ...+... | semmle.label | ...+... | subpaths #select -| TaintedPath.go:16:29:16:40 | tainted_path | TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:16:29:16:40 | tainted_path | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | -| TaintedPath.go:20:28:20:69 | call to Join | TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:20:28:20:69 | call to Join | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | -| TaintedPath.go:67:28:67:57 | call to Clean | TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:67:28:67:57 | call to Clean | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | -| TaintedPath.go:77:28:77:56 | call to Base | TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:77:28:77:56 | call to Base | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | -| tst.go:17:41:17:56 | selection of Filename | tst.go:14:2:14:39 | ... := ...[1] : pointer type | tst.go:17:41:17:56 | selection of Filename | This path depends on a $@. | tst.go:14:2:14:39 | ... := ...[1] | user-provided value | +| TaintedPath.go:16:29:16:40 | tainted_path | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:16:29:16:40 | tainted_path | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | +| TaintedPath.go:20:28:20:69 | call to Join | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:20:28:20:69 | call to Join | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | +| TaintedPath.go:67:28:67:57 | call to Clean | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:67:28:67:57 | call to Clean | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | +| TaintedPath.go:77:28:77:56 | call to Base | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:77:28:77:56 | call to Base | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | From c4c0b22bc78075dfa0463f6fc8b09b9e3c1ef34c Mon Sep 17 00:00:00 2001 From: Kevin Stubbings Date: Sun, 8 Oct 2023 16:37:35 -0700 Subject: [PATCH 04/97] Formattinga and change notes --- .../2023-10-08-addional-gopath-sanitizers.md | 5 +++++ .../go/security/TaintedPathCustomizations.qll | 15 ++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 go/ql/lib/change-notes/2023-10-08-addional-gopath-sanitizers.md diff --git a/go/ql/lib/change-notes/2023-10-08-addional-gopath-sanitizers.md b/go/ql/lib/change-notes/2023-10-08-addional-gopath-sanitizers.md new file mode 100644 index 00000000000..0666fee50d5 --- /dev/null +++ b/go/ql/lib/change-notes/2023-10-08-addional-gopath-sanitizers.md @@ -0,0 +1,5 @@ + +* --- +category: minorAnalysis +--- +* Added filepath.Base, strings.ReplaceAll, http.ParseMultipartForm sanitizers and remove path sanitizer. \ No newline at end of file diff --git a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll index 815a628fec3..c6eec12d9ef 100644 --- a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll +++ b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll @@ -6,6 +6,7 @@ import go import semmle.go.dataflow.barrierguardutil.RegexpCheck import DataFlow + /** * Provides extension points for customizing the taint tracking configuration for reasoning about * path-traversal vulnerabilities. @@ -79,15 +80,15 @@ module TaintedPath { class FilepathCleanSanitizer extends Sanitizer { FilepathCleanSanitizer() { exists(DataFlow::CallNode cleanCall, StringOps::Concatenation concatNode | - cleanCall = - any(Function f | f.hasQualifiedName("path/filepath", "Clean")).getACall() and + cleanCall = any(Function f | f.hasQualifiedName("path/filepath", "Clean")).getACall() and concatNode = cleanCall.getArgument(0) and concatNode.getOperand(0).asExpr().(StringLit).getValue() = "/" and this = cleanCall.getResult() ) } } - /** + + /** * A call to `filepath.Base(e)`, considered to sanitize `e` against path traversal. */ class FilepathBaseSanitizer extends Sanitizer { @@ -107,8 +108,8 @@ module TaintedPath { frn.getField().hasQualifiedName("mime/multipart", "FileHeader", "Filename") and this = frn ) - } } + } /** * A check of the form `!strings.Contains(nd, "..")`, considered as a sanitizer guard for @@ -127,15 +128,15 @@ module TaintedPath { branch = false } } -/** + + /** * A replacement of the form `!strings.ReplaceAll(nd, "..")` or `!strings.ReplaceAll(nd, ".")`, considered as a sanitizer for * path traversal. */ class DotDotReplace extends Sanitizer { DotDotReplace() { exists(DataFlow::CallNode cleanCall, DataFlow::Node valueNode | - cleanCall = - any(Function f | f.hasQualifiedName("strings", "ReplaceAll")).getACall() and + cleanCall = any(Function f | f.hasQualifiedName("strings", "ReplaceAll")).getACall() and valueNode = cleanCall.getArgument(1) and valueNode.asExpr().(StringLit).getValue() = ["..", "."] and this = cleanCall.getResult() From 30fe4168e34cc4059c59cabcee77a30e850c7f7c Mon Sep 17 00:00:00 2001 From: Kevin Stubbings Date: Mon, 11 Mar 2024 15:08:10 -0700 Subject: [PATCH 05/97] Removed filepath.base sanitizer --- .../2023-10-08-addional-gopath-sanitizers.md | 5 ---- .../2024-03-11-addional-gopath-sanitizers.md | 5 ++++ .../go/security/TaintedPathCustomizations.qll | 13 ---------- .../Security/CWE-022/TaintedPath.expected | 25 ++----------------- .../Security/CWE-022/TaintedPath.go | 10 -------- 5 files changed, 7 insertions(+), 51 deletions(-) delete mode 100644 go/ql/lib/change-notes/2023-10-08-addional-gopath-sanitizers.md create mode 100644 go/ql/lib/change-notes/2024-03-11-addional-gopath-sanitizers.md diff --git a/go/ql/lib/change-notes/2023-10-08-addional-gopath-sanitizers.md b/go/ql/lib/change-notes/2023-10-08-addional-gopath-sanitizers.md deleted file mode 100644 index 0666fee50d5..00000000000 --- a/go/ql/lib/change-notes/2023-10-08-addional-gopath-sanitizers.md +++ /dev/null @@ -1,5 +0,0 @@ - -* --- -category: minorAnalysis ---- -* Added filepath.Base, strings.ReplaceAll, http.ParseMultipartForm sanitizers and remove path sanitizer. \ No newline at end of file diff --git a/go/ql/lib/change-notes/2024-03-11-addional-gopath-sanitizers.md b/go/ql/lib/change-notes/2024-03-11-addional-gopath-sanitizers.md new file mode 100644 index 00000000000..acfafc26cc5 --- /dev/null +++ b/go/ql/lib/change-notes/2024-03-11-addional-gopath-sanitizers.md @@ -0,0 +1,5 @@ + +* --- +category: minorAnalysis +--- +* Added strings.ReplaceAll, http.ParseMultipartForm sanitizers and remove path sanitizer. \ No newline at end of file diff --git a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll index c6eec12d9ef..70f279d307b 100644 --- a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll +++ b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll @@ -88,19 +88,6 @@ module TaintedPath { } } - /** - * A call to `filepath.Base(e)`, considered to sanitize `e` against path traversal. - */ - class FilepathBaseSanitizer extends Sanitizer { - FilepathBaseSanitizer() { - exists(Function f, FunctionOutput outp | - f.hasQualifiedName("path/filepath", "Base") and - outp.isResult(0) and - this = outp.getNode(f.getACall()) - ) - } - } - /**An call to ParseMultipartForm creates multipart.Form and cleans mutlpart.Form.FileHeader.Filename using path.Base() */ class MultipartClean extends Sanitizer { MultipartClean() { diff --git a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected index c3a4fc5a616..4b4748f0d9c 100644 --- a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected +++ b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected @@ -1,28 +1,10 @@ edges -<<<<<<< HEAD -<<<<<<< HEAD | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:13:18:13:30 | call to Query | provenance | | | TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:16:29:16:40 | tainted_path | provenance | | | TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:20:57:20:68 | tainted_path | provenance | | +| TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:67:39:67:56 | ...+... | provenance | | | TaintedPath.go:20:57:20:68 | tainted_path | TaintedPath.go:20:28:20:69 | call to Join | provenance | | -| tst.go:14:2:14:39 | ... := ...[1] | tst.go:17:41:17:56 | selection of Filename | provenance | | -======= -| TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:16:29:16:40 | tainted_path | -| TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:20:28:20:69 | call to Join | -| TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:67:28:67:57 | call to Clean | -| TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:77:28:77:56 | call to Base | -| tst.go:14:2:14:39 | ... := ...[1] : pointer type | tst.go:17:41:17:56 | selection of Filename | ->>>>>>> a45343fb6c (Add New Sanitizers and Modify Old Ones) -======= -| TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:13:18:13:30 | call to Query | -| TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:16:29:16:40 | tainted_path | -| TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:20:57:20:68 | tainted_path | -| TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:67:39:67:56 | ...+... | -| TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:77:38:77:55 | ...+... | -| TaintedPath.go:20:57:20:68 | tainted_path | TaintedPath.go:20:28:20:69 | call to Join | -| TaintedPath.go:67:39:67:56 | ...+... | TaintedPath.go:67:28:67:57 | call to Clean | -| TaintedPath.go:77:38:77:55 | ...+... | TaintedPath.go:77:28:77:56 | call to Base | ->>>>>>> db14838a4f (resolve feedback) +| TaintedPath.go:67:39:67:56 | ...+... | TaintedPath.go:67:28:67:57 | call to Clean | provenance | | nodes | TaintedPath.go:13:18:13:22 | selection of URL | semmle.label | selection of URL | | TaintedPath.go:13:18:13:30 | call to Query | semmle.label | call to Query | @@ -31,11 +13,8 @@ nodes | TaintedPath.go:20:57:20:68 | tainted_path | semmle.label | tainted_path | | TaintedPath.go:67:28:67:57 | call to Clean | semmle.label | call to Clean | | TaintedPath.go:67:39:67:56 | ...+... | semmle.label | ...+... | -| TaintedPath.go:77:28:77:56 | call to Base | semmle.label | call to Base | -| TaintedPath.go:77:38:77:55 | ...+... | semmle.label | ...+... | subpaths #select | TaintedPath.go:16:29:16:40 | tainted_path | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:16:29:16:40 | tainted_path | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | | TaintedPath.go:20:28:20:69 | call to Join | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:20:28:20:69 | call to Join | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | | TaintedPath.go:67:28:67:57 | call to Clean | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:67:28:67:57 | call to Clean | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | -| TaintedPath.go:77:28:77:56 | call to Base | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:77:28:77:56 | call to Base | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | diff --git a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go index 629b7556e9b..378cc3acea5 100644 --- a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go +++ b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go @@ -67,16 +67,6 @@ func handler(w http.ResponseWriter, r *http.Request) { data, _ = ioutil.ReadFile(path.Clean("/" + tainted_path)) w.Write(data) - // GOOD: Sanitized by filepath.Base with a prepended '/' forcing interpretation - // as an absolute path, so that Base will throw away any leading `..` components. - data, _ = ioutil.ReadFile(filepath.Base("/" + tainted_path)) - w.Write(data) - - // BAD: Sanitized by path.Base with a prepended '/' forcing interpretation - // as an absolute path, however is not sufficient for Windows paths. - data, _ = ioutil.ReadFile(path.Base("/" + tainted_path)) - w.Write(data) - // GOOD: Multipart.Form.FileHeader.Filename sanitized by filepath.Base when calling ParseMultipartForm r.ParseMultipartForm(32 << 20) form := r.MultipartForm From 5acc15bfff6c3d9e6bb2e9a6197d34a971fec38a Mon Sep 17 00:00:00 2001 From: Kevin Stubbings Date: Mon, 11 Mar 2024 20:13:08 -0700 Subject: [PATCH 06/97] fix grammar --- go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll index 70f279d307b..5959bc0ffe8 100644 --- a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll +++ b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll @@ -88,7 +88,7 @@ module TaintedPath { } } - /**An call to ParseMultipartForm creates multipart.Form and cleans mutlpart.Form.FileHeader.Filename using path.Base() */ + /**An call to ParseMultipartForm creates multipart.Form and cleans multipart.Form.FileHeader.Filename using path.Base() */ class MultipartClean extends Sanitizer { MultipartClean() { exists(DataFlow::FieldReadNode frn | From 33c17313b413e9f742f825e5bc4c65609f0d1c0a Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Tue, 12 Mar 2024 11:59:10 +0000 Subject: [PATCH 07/97] Add test for not extracting values for intermediate string concatenations --- .../no-intermediate-strings/tst.expected | 11 +++++++++++ .../no-intermediate-strings/tst.go | 5 +++++ .../no-intermediate-strings/tst.ql | 17 +++++++++++++++++ 3 files changed, 33 insertions(+) create mode 100644 go/ql/test/extractor-tests/no-intermediate-strings/tst.expected create mode 100644 go/ql/test/extractor-tests/no-intermediate-strings/tst.go create mode 100644 go/ql/test/extractor-tests/no-intermediate-strings/tst.ql diff --git a/go/ql/test/extractor-tests/no-intermediate-strings/tst.expected b/go/ql/test/extractor-tests/no-intermediate-strings/tst.expected new file mode 100644 index 00000000000..9746d74fb90 --- /dev/null +++ b/go/ql/test/extractor-tests/no-intermediate-strings/tst.expected @@ -0,0 +1,11 @@ +| tst.go:4:6:4:8 | "a" | a | +| tst.go:4:6:4:14 | ...+... | | +| tst.go:4:6:4:20 | ...+... | | +| tst.go:4:6:4:26 | ...+... | | +| tst.go:4:6:4:32 | ...+... | | +| tst.go:4:6:4:38 | ...+... | abcdef | +| tst.go:4:12:4:14 | "b" | b | +| tst.go:4:18:4:20 | "c" | c | +| tst.go:4:24:4:26 | "d" | d | +| tst.go:4:30:4:32 | "e" | e | +| tst.go:4:36:4:38 | "f" | f | diff --git a/go/ql/test/extractor-tests/no-intermediate-strings/tst.go b/go/ql/test/extractor-tests/no-intermediate-strings/tst.go new file mode 100644 index 00000000000..c79c97a8e88 --- /dev/null +++ b/go/ql/test/extractor-tests/no-intermediate-strings/tst.go @@ -0,0 +1,5 @@ +package main + +func main() { + _ = "a" + "b" + "c" + "d" + "e" + "f" +} diff --git a/go/ql/test/extractor-tests/no-intermediate-strings/tst.ql b/go/ql/test/extractor-tests/no-intermediate-strings/tst.ql new file mode 100644 index 00000000000..6367ef51e70 --- /dev/null +++ b/go/ql/test/extractor-tests/no-intermediate-strings/tst.ql @@ -0,0 +1,17 @@ +import go + +string checkStringValue(Expr e) { + result = e.getStringValue() + or + not exists(e.getStringValue()) and result = "" +} + +from Expr e +where e.getType() instanceof StringType +// We should get string values for `"a"`, `"b"`, `"c"` and `"a" + "b" + "c" +// but not `"a" + "b"`. In the extractor we avoid storing the value of +// intermediate strings in string concatenations because in pathological cases +// this could lead to a quadratic blowup in the size of string values stored, +// which then causes performance problems when we iterate through all string +// values. +select e, checkStringValue(e) From fc8caa66c8ac4cb9b6a9a31f5ea24d0b50d3297a Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Fri, 23 Feb 2024 11:35:50 +0100 Subject: [PATCH 08/97] Python: Prepare for general content in type-tracker Due to the char-pred of Content, this change should keep exactly the same behavior as before. --- .../python/dataflow/new/TypeTracking.qll | 14 +++--- .../new/internal/TypeTrackingImpl.qll | 45 +++++-------------- 2 files changed, 21 insertions(+), 38 deletions(-) diff --git a/python/ql/lib/semmle/python/dataflow/new/TypeTracking.qll b/python/ql/lib/semmle/python/dataflow/new/TypeTracking.qll index 4f1810f059e..9d0bcb3c487 100644 --- a/python/ql/lib/semmle/python/dataflow/new/TypeTracking.qll +++ b/python/ql/lib/semmle/python/dataflow/new/TypeTracking.qll @@ -5,6 +5,7 @@ private import internal.TypeTrackingImpl as Impl import Impl::Shared::TypeTracking +private import semmle.python.dataflow.new.internal.DataFlowPublic as DataFlowPublic /** A string that may appear as the name of an attribute or access path. */ class AttributeName = Impl::TypeTrackingInput::Content; @@ -40,7 +41,11 @@ class TypeTracker extends Impl::TypeTracker { * Holds if this is the starting point of type tracking, and the value starts in the attribute named `attrName`. * The type tracking only ends after the attribute has been loaded. */ - predicate startInAttr(string attrName) { this.startInContent(attrName) } + predicate startInAttr(string attrName) { + exists(DataFlowPublic::AttributeContent content | content.getAttribute() = attrName | + this.startInContent(content) + ) + } /** * INTERNAL. DO NOT USE. @@ -48,9 +53,8 @@ class TypeTracker extends Impl::TypeTracker { * Gets the attribute associated with this type tracker. */ string getAttr() { - result = this.getContent().asSome() - or - this.getContent().isNone() and - result = "" + if this.getContent().asSome() instanceof DataFlowPublic::AttributeContent + then result = this.getContent().asSome().(DataFlowPublic::AttributeContent).getAttribute() + else result = "" } } diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll b/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll index 1a9bdb5202e..8b6e53c8b74 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll @@ -97,24 +97,14 @@ private module SummaryTypeTrackerInput implements SummaryTypeTracker::Input { private module TypeTrackerSummaryFlow = SummaryTypeTracker::SummaryFlow; -/** - * Gets the name of a possible piece of content. For Python, this is currently only attribute names, - * using the name of the attribute for the corresponding content. - */ -private string getPossibleContentName() { - Stages::TypeTracking::ref() and // the TypeTracking::append() etc. predicates that we want to cache depend on this predicate, so we can place the `ref()` call here to get around identical files. - result = any(DataFlowPublic::AttrRef a).getAttributeName() -} - module TypeTrackingInput implements Shared::TypeTrackingInput { class Node = DataFlowPublic::Node; class LocalSourceNode = DataFlowPublic::LocalSourceNode; - class Content instanceof string { - Content() { this = getPossibleContentName() } - - string toString() { result = this } + class Content extends DataFlowPublic::Content { + // this char-pred is just a temporary restriction while transitioning to more general content + Content() { this instanceof DataFlowPublic::AttributeContent } } /** @@ -181,46 +171,35 @@ module TypeTrackingInput implements Shared::TypeTrackingInput { * Holds if `nodeFrom` is being written to the `content` content of the object in `nodeTo`. */ predicate storeStep(Node nodeFrom, Node nodeTo, Content content) { - exists(DataFlowPublic::AttrWrite a | - a.mayHaveAttributeName(content) and + exists(DataFlowPublic::AttrWrite a, string attrName | + content.(DataFlowPublic::AttributeContent).getAttribute() = attrName and + a.mayHaveAttributeName(attrName) and nodeFrom = a.getValue() and nodeTo = a.getObject() ) or - exists(DataFlowPublic::ContentSet contents | - contents.(DataFlowPublic::AttributeContent).getAttribute() = content - | - TypeTrackerSummaryFlow::basicStoreStep(nodeFrom, nodeTo, contents) - ) + TypeTrackerSummaryFlow::basicStoreStep(nodeFrom, nodeTo, content) } /** * Holds if `nodeTo` is the result of accessing the `content` content of `nodeFrom`. */ predicate loadStep(Node nodeFrom, LocalSourceNode nodeTo, Content content) { - exists(DataFlowPublic::AttrRead a | - a.mayHaveAttributeName(content) and + exists(DataFlowPublic::AttrRead a, string attrName | + content.(DataFlowPublic::AttributeContent).getAttribute() = attrName and + a.mayHaveAttributeName(attrName) and nodeFrom = a.getObject() and nodeTo = a ) or - exists(DataFlowPublic::ContentSet contents | - contents.(DataFlowPublic::AttributeContent).getAttribute() = content - | - TypeTrackerSummaryFlow::basicLoadStep(nodeFrom, nodeTo, contents) - ) + TypeTrackerSummaryFlow::basicLoadStep(nodeFrom, nodeTo, content) } /** * Holds if the `loadContent` of `nodeFrom` is stored in the `storeContent` of `nodeTo`. */ predicate loadStoreStep(Node nodeFrom, Node nodeTo, Content loadContent, Content storeContent) { - exists(DataFlowPublic::ContentSet loadContents, DataFlowPublic::ContentSet storeContents | - loadContents.(DataFlowPublic::AttributeContent).getAttribute() = loadContent and - storeContents.(DataFlowPublic::AttributeContent).getAttribute() = storeContent - | - TypeTrackerSummaryFlow::basicLoadStoreStep(nodeFrom, nodeTo, loadContents, storeContents) - ) + TypeTrackerSummaryFlow::basicLoadStoreStep(nodeFrom, nodeTo, loadContent, storeContent) } /** From 636cf611ae8d5f9ac3025bacff6e028902e89f88 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Fri, 23 Feb 2024 11:36:47 +0100 Subject: [PATCH 09/97] Python: Allow general content in type-tracker This should not result in many changes, since store/load steps are still only implemented for attributes. --- .../semmle/python/dataflow/new/internal/TypeTrackingImpl.qll | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll b/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll index 8b6e53c8b74..81c1f369561 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll @@ -102,10 +102,7 @@ module TypeTrackingInput implements Shared::TypeTrackingInput { class LocalSourceNode = DataFlowPublic::LocalSourceNode; - class Content extends DataFlowPublic::Content { - // this char-pred is just a temporary restriction while transitioning to more general content - Content() { this instanceof DataFlowPublic::AttributeContent } - } + class Content = DataFlowPublic::Content; /** * A label to use for `WithContent` and `WithoutContent` steps, restricting From 7721fb33314cee322fd4b7ac3c8bd5bfe8b2e353 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Fri, 23 Feb 2024 11:44:08 +0100 Subject: [PATCH 10/97] Python: Setup shared read/store steps --- .../dataflow/new/internal/DataFlowPrivate.qll | 14 ++++++++++++++ .../dataflow/new/internal/TypeTrackingImpl.qll | 4 ++++ 2 files changed, 18 insertions(+) diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll index 47f41d0cd05..5ccfa251634 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll @@ -641,11 +641,18 @@ predicate jumpStepNotSharedWithTypeTracker(Node nodeFrom, Node nodeTo) { //-------- // Field flow //-------- +/** + * Subset of `storeStep` that should be shared with type-tracking. + */ +predicate storeStepCommon(Node nodeFrom, ContentSet c, Node nodeTo) { none() } + /** * Holds if data can flow from `nodeFrom` to `nodeTo` via an assignment to * content `c`. */ predicate storeStep(Node nodeFrom, ContentSet c, Node nodeTo) { + storeStepCommon(nodeFrom, c, nodeTo) + or listStoreStep(nodeFrom, c, nodeTo) or setStoreStep(nodeFrom, c, nodeTo) @@ -891,10 +898,17 @@ predicate attributeStoreStep(Node nodeFrom, AttributeContent c, Node nodeTo) { ) } +/** + * Subset of `readStep` that should be shared with type-tracking. + */ +predicate readStepCommon(Node nodeFrom, ContentSet c, Node nodeTo) { none() } + /** * Holds if data can flow from `nodeFrom` to `nodeTo` via a read of content `c`. */ predicate readStep(Node nodeFrom, ContentSet c, Node nodeTo) { + readStepCommon(nodeFrom, c, nodeTo) + or subscriptReadStep(nodeFrom, c, nodeTo) or iterableUnpackingReadStep(nodeFrom, c, nodeTo) diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll b/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll index 81c1f369561..68779208de9 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll @@ -175,6 +175,8 @@ module TypeTrackingInput implements Shared::TypeTrackingInput { nodeTo = a.getObject() ) or + DataFlowPrivate::storeStepCommon(nodeFrom, content, nodeTo) + or TypeTrackerSummaryFlow::basicStoreStep(nodeFrom, nodeTo, content) } @@ -189,6 +191,8 @@ module TypeTrackingInput implements Shared::TypeTrackingInput { nodeTo = a ) or + DataFlowPrivate::readStepCommon(nodeFrom, content, nodeTo) + or TypeTrackerSummaryFlow::basicLoadStep(nodeFrom, nodeTo, content) } From a95bb7c86b1b9047fd5bcc6adf77ee195074a8c5 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Fri, 23 Feb 2024 13:24:48 +0100 Subject: [PATCH 11/97] Python: Expand function reference in content test --- .../CallGraph/InlineCallGraphTest.expected | 3 +- .../CallGraph/code/func_ref_in_content.py | 53 +++++++++++++++++++ .../CallGraph/code/tuple_function_return.py | 15 ------ 3 files changed, 55 insertions(+), 16 deletions(-) create mode 100644 python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py delete mode 100644 python/ql/test/experimental/library-tests/CallGraph/code/tuple_function_return.py diff --git a/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected b/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected index 55774486be0..504c5251a1a 100644 --- a/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected +++ b/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected @@ -15,8 +15,9 @@ pointsTo_found_typeTracker_notFound | code/func_defined_outside_class.py:39:11:39:21 | ControlFlowNode for _gen() | B._gen | | code/func_defined_outside_class.py:42:1:42:7 | ControlFlowNode for Attribute() | B._gen.func | | code/func_defined_outside_class.py:43:1:43:7 | ControlFlowNode for Attribute() | B._gen.func | +| code/func_ref_in_content.py:17:1:17:4 | ControlFlowNode for f2() | func | +| code/func_ref_in_content.py:20:1:20:4 | ControlFlowNode for f3() | func | | code/funky_regression.py:15:9:15:17 | ControlFlowNode for Attribute() | Wat.f2 | -| code/tuple_function_return.py:15:1:15:4 | ControlFlowNode for f2() | func | | code/type_tracking_limitation.py:8:1:8:3 | ControlFlowNode for x() | my_func | typeTracker_found_pointsTo_notFound | code/callable_as_argument.py:29:5:29:12 | ControlFlowNode for Attribute() | test_class.InsideTestFunc.sm | diff --git a/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py b/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py new file mode 100644 index 00000000000..b89a013f5b1 --- /dev/null +++ b/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py @@ -0,0 +1,53 @@ +def func(): + print("func()") + +def return_func(): + return func + +f1 = return_func() # $ pt,tt=return_func +f1() # $ pt,tt=func + + +def return_func_in_tuple(): + return (func, 42) + +tup = return_func_in_tuple() # $ pt,tt=return_func_in_tuple + +f2, _ = tup +f2() # $ pt=func MISSING: tt + +f3 = tup[0] +f3() # $ pt=func MISSING: tt + + +def return_func_in_dict(): + return {'func': func, 'val': 42} + +dct = return_func_in_dict() # $ pt,tt=return_func_in_dict + +f4 = dct['func'] +f4() # $ MISSING: tt=func + + +def return_func_in_dict_update(): + d = {} + d["func"] = func + return d + +dct2 = return_func_in_dict_update() # $ pt,tt=return_func_in_dict_update + +f5 = dct2['func'] +f5() # $ MISSING: tt=func + + +def return_func_in_list(): + return [func, 42] + +lst = return_func_in_list() # $ pt,tt=return_func_in_list + +f6 = lst[0] +f6() # $ MISSING: pt,tt=func + +if eval("False"): # don't run this, but fool analysis to still consider it (doesn't wok if you just to `if False:`) + f7 = lst[1] + f7() diff --git a/python/ql/test/experimental/library-tests/CallGraph/code/tuple_function_return.py b/python/ql/test/experimental/library-tests/CallGraph/code/tuple_function_return.py deleted file mode 100644 index f87b1aa23e8..00000000000 --- a/python/ql/test/experimental/library-tests/CallGraph/code/tuple_function_return.py +++ /dev/null @@ -1,15 +0,0 @@ -def func(): - print("func()") - -def return_func(): - return func - -def return_func_in_tuple(): - return (func, 42) - -f1 = return_func() # $ pt,tt=return_func -f1() # $ pt,tt=func - - -f2, _ = return_func_in_tuple() # $ pt,tt=return_func_in_tuple -f2() # $ pt=func MISSING: tt From ece8245a4be1f05fffc2e9cd7acd5e387729538e Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Fri, 23 Feb 2024 13:39:49 +0100 Subject: [PATCH 12/97] Python: type-track through tuple content --- .../python/dataflow/new/internal/DataFlowPrivate.qll | 12 ++++++------ .../CallGraph/InlineCallGraphTest.expected | 1 - .../CallGraph/code/func_ref_in_content.py | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll index 5ccfa251634..22fb979f9dc 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll @@ -644,7 +644,9 @@ predicate jumpStepNotSharedWithTypeTracker(Node nodeFrom, Node nodeTo) { /** * Subset of `storeStep` that should be shared with type-tracking. */ -predicate storeStepCommon(Node nodeFrom, ContentSet c, Node nodeTo) { none() } +predicate storeStepCommon(Node nodeFrom, ContentSet c, Node nodeTo) { + tupleStoreStep(nodeFrom, c, nodeTo) +} /** * Holds if data can flow from `nodeFrom` to `nodeTo` via an assignment to @@ -657,8 +659,6 @@ predicate storeStep(Node nodeFrom, ContentSet c, Node nodeTo) { or setStoreStep(nodeFrom, c, nodeTo) or - tupleStoreStep(nodeFrom, c, nodeTo) - or dictStoreStep(nodeFrom, c, nodeTo) or moreDictStoreSteps(nodeFrom, c, nodeTo) @@ -901,7 +901,9 @@ predicate attributeStoreStep(Node nodeFrom, AttributeContent c, Node nodeTo) { /** * Subset of `readStep` that should be shared with type-tracking. */ -predicate readStepCommon(Node nodeFrom, ContentSet c, Node nodeTo) { none() } +predicate readStepCommon(Node nodeFrom, ContentSet c, Node nodeTo) { + subscriptReadStep(nodeFrom, c, nodeTo) +} /** * Holds if data can flow from `nodeFrom` to `nodeTo` via a read of content `c`. @@ -909,8 +911,6 @@ predicate readStepCommon(Node nodeFrom, ContentSet c, Node nodeTo) { none() } predicate readStep(Node nodeFrom, ContentSet c, Node nodeTo) { readStepCommon(nodeFrom, c, nodeTo) or - subscriptReadStep(nodeFrom, c, nodeTo) - or iterableUnpackingReadStep(nodeFrom, c, nodeTo) or matchReadStep(nodeFrom, c, nodeTo) diff --git a/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected b/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected index 504c5251a1a..667ebf28d75 100644 --- a/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected +++ b/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected @@ -16,7 +16,6 @@ pointsTo_found_typeTracker_notFound | code/func_defined_outside_class.py:42:1:42:7 | ControlFlowNode for Attribute() | B._gen.func | | code/func_defined_outside_class.py:43:1:43:7 | ControlFlowNode for Attribute() | B._gen.func | | code/func_ref_in_content.py:17:1:17:4 | ControlFlowNode for f2() | func | -| code/func_ref_in_content.py:20:1:20:4 | ControlFlowNode for f3() | func | | code/funky_regression.py:15:9:15:17 | ControlFlowNode for Attribute() | Wat.f2 | | code/type_tracking_limitation.py:8:1:8:3 | ControlFlowNode for x() | my_func | typeTracker_found_pointsTo_notFound diff --git a/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py b/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py index b89a013f5b1..87abb4198e9 100644 --- a/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py +++ b/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py @@ -17,7 +17,7 @@ f2, _ = tup f2() # $ pt=func MISSING: tt f3 = tup[0] -f3() # $ pt=func MISSING: tt +f3() # $ tt,pt=func def return_func_in_dict(): From 73fe596753357032044842f90816421ddd16490c Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Fri, 23 Feb 2024 13:57:57 +0100 Subject: [PATCH 13/97] Python: type-tracking through dictionary construction --- .../semmle/python/dataflow/new/internal/DataFlowPrivate.qll | 4 ++-- .../library-tests/CallGraph/InlineCallGraphTest.expected | 1 + .../library-tests/CallGraph/code/func_ref_in_content.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll index 22fb979f9dc..3b589da37d6 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll @@ -646,6 +646,8 @@ predicate jumpStepNotSharedWithTypeTracker(Node nodeFrom, Node nodeTo) { */ predicate storeStepCommon(Node nodeFrom, ContentSet c, Node nodeTo) { tupleStoreStep(nodeFrom, c, nodeTo) + or + dictStoreStep(nodeFrom, c, nodeTo) } /** @@ -659,8 +661,6 @@ predicate storeStep(Node nodeFrom, ContentSet c, Node nodeTo) { or setStoreStep(nodeFrom, c, nodeTo) or - dictStoreStep(nodeFrom, c, nodeTo) - or moreDictStoreSteps(nodeFrom, c, nodeTo) or comprehensionStoreStep(nodeFrom, c, nodeTo) diff --git a/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected b/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected index 667ebf28d75..ab97d594a4e 100644 --- a/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected +++ b/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected @@ -38,6 +38,7 @@ typeTracker_found_pointsTo_notFound | code/class_super.py:101:1:101:7 | ControlFlowNode for Attribute() | Z.foo | | code/class_super.py:108:1:108:8 | ControlFlowNode for Attribute() | Z.foo | | code/def_in_function.py:22:5:22:11 | ControlFlowNode for Attribute() | test.A.foo | +| code/func_ref_in_content.py:29:1:29:4 | ControlFlowNode for f4() | func | | code/isinstance.py:9:13:9:22 | ControlFlowNode for Attribute() | A.foo | | code/isinstance.py:9:13:9:22 | ControlFlowNode for Attribute() | ASub.foo | | code/isinstance.py:14:13:14:22 | ControlFlowNode for Attribute() | A.foo | diff --git a/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py b/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py index 87abb4198e9..57b11915c51 100644 --- a/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py +++ b/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py @@ -26,7 +26,7 @@ def return_func_in_dict(): dct = return_func_in_dict() # $ pt,tt=return_func_in_dict f4 = dct['func'] -f4() # $ MISSING: tt=func +f4() # $ tt=func def return_func_in_dict_update(): From dac2b57bb029e345f79e0337f5ab1ba2a2137db7 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Fri, 23 Feb 2024 14:51:38 +0100 Subject: [PATCH 14/97] Python: type-track through dict-updates --- .../dataflow/new/internal/DataFlowPrivate.qll | 4 ++-- .../dataflow/new/internal/TypeTrackingImpl.qll | 13 ++++++++++++- .../CallGraph/InlineCallGraphTest.expected | 1 + .../CallGraph/code/func_ref_in_content.py | 2 +- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll index 3b589da37d6..98841726a74 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll @@ -648,6 +648,8 @@ predicate storeStepCommon(Node nodeFrom, ContentSet c, Node nodeTo) { tupleStoreStep(nodeFrom, c, nodeTo) or dictStoreStep(nodeFrom, c, nodeTo) + or + moreDictStoreSteps(nodeFrom, c, nodeTo) } /** @@ -661,8 +663,6 @@ predicate storeStep(Node nodeFrom, ContentSet c, Node nodeTo) { or setStoreStep(nodeFrom, c, nodeTo) or - moreDictStoreSteps(nodeFrom, c, nodeTo) - or comprehensionStoreStep(nodeFrom, c, nodeTo) or iterableUnpackingStoreStep(nodeFrom, c, nodeTo) diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll b/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll index 68779208de9..8b3e1a95ef1 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll @@ -175,7 +175,18 @@ module TypeTrackingInput implements Shared::TypeTrackingInput { nodeTo = a.getObject() ) or - DataFlowPrivate::storeStepCommon(nodeFrom, content, nodeTo) + // type-tracking doesn't really handle PostUpdateNodes, so for some assignment steps + // like `my_dict["foo"] = foo` the data-flow step targets the PostUpdateNode for + // `my_dict`, where we want to translate that into a type-tracking step that targets + // the normal/non-PostUpdateNode for `my_dict`. + exists(DataFlowPublic::Node storeTarget | + DataFlowPrivate::storeStepCommon(nodeFrom, content, storeTarget) + | + not storeTarget instanceof DataFlowPrivate::SyntheticPostUpdateNode and + nodeTo = storeTarget + or + nodeTo = storeTarget.(DataFlowPrivate::SyntheticPostUpdateNode).getPreUpdateNode() + ) or TypeTrackerSummaryFlow::basicStoreStep(nodeFrom, nodeTo, content) } diff --git a/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected b/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected index ab97d594a4e..378b2c64957 100644 --- a/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected +++ b/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected @@ -39,6 +39,7 @@ typeTracker_found_pointsTo_notFound | code/class_super.py:108:1:108:8 | ControlFlowNode for Attribute() | Z.foo | | code/def_in_function.py:22:5:22:11 | ControlFlowNode for Attribute() | test.A.foo | | code/func_ref_in_content.py:29:1:29:4 | ControlFlowNode for f4() | func | +| code/func_ref_in_content.py:40:1:40:4 | ControlFlowNode for f5() | func | | code/isinstance.py:9:13:9:22 | ControlFlowNode for Attribute() | A.foo | | code/isinstance.py:9:13:9:22 | ControlFlowNode for Attribute() | ASub.foo | | code/isinstance.py:14:13:14:22 | ControlFlowNode for Attribute() | A.foo | diff --git a/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py b/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py index 57b11915c51..4bea545cb0f 100644 --- a/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py +++ b/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py @@ -37,7 +37,7 @@ def return_func_in_dict_update(): dct2 = return_func_in_dict_update() # $ pt,tt=return_func_in_dict_update f5 = dct2['func'] -f5() # $ MISSING: tt=func +f5() # $ tt=func def return_func_in_list(): From 0cf3fe4a4c5d7772d2efefa0d45cf2d89e46c160 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Fri, 23 Feb 2024 14:59:07 +0100 Subject: [PATCH 15/97] Python: Expand dict update tests --- .../CallGraph/InlineCallGraphTest.expected | 8 +++++--- .../CallGraph/code/func_ref_in_content.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected b/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected index 378b2c64957..c5b7d6dc473 100644 --- a/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected +++ b/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected @@ -15,7 +15,7 @@ pointsTo_found_typeTracker_notFound | code/func_defined_outside_class.py:39:11:39:21 | ControlFlowNode for _gen() | B._gen | | code/func_defined_outside_class.py:42:1:42:7 | ControlFlowNode for Attribute() | B._gen.func | | code/func_defined_outside_class.py:43:1:43:7 | ControlFlowNode for Attribute() | B._gen.func | -| code/func_ref_in_content.py:17:1:17:4 | ControlFlowNode for f2() | func | +| code/func_ref_in_content.py:20:1:20:4 | ControlFlowNode for f2() | func | | code/funky_regression.py:15:9:15:17 | ControlFlowNode for Attribute() | Wat.f2 | | code/type_tracking_limitation.py:8:1:8:3 | ControlFlowNode for x() | my_func | typeTracker_found_pointsTo_notFound @@ -38,8 +38,10 @@ typeTracker_found_pointsTo_notFound | code/class_super.py:101:1:101:7 | ControlFlowNode for Attribute() | Z.foo | | code/class_super.py:108:1:108:8 | ControlFlowNode for Attribute() | Z.foo | | code/def_in_function.py:22:5:22:11 | ControlFlowNode for Attribute() | test.A.foo | -| code/func_ref_in_content.py:29:1:29:4 | ControlFlowNode for f4() | func | -| code/func_ref_in_content.py:40:1:40:4 | ControlFlowNode for f5() | func | +| code/func_ref_in_content.py:32:1:32:4 | ControlFlowNode for f4() | func | +| code/func_ref_in_content.py:46:1:46:4 | ControlFlowNode for f5() | func | +| code/func_ref_in_content.py:48:1:48:15 | ControlFlowNode for Subscript() | func2 | +| code/func_ref_in_content.py:50:1:50:19 | ControlFlowNode for Subscript() | func2 | | code/isinstance.py:9:13:9:22 | ControlFlowNode for Attribute() | A.foo | | code/isinstance.py:9:13:9:22 | ControlFlowNode for Attribute() | ASub.foo | | code/isinstance.py:14:13:14:22 | ControlFlowNode for Attribute() | A.foo | diff --git a/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py b/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py index 4bea545cb0f..b249ec0b2e9 100644 --- a/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py +++ b/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py @@ -1,6 +1,9 @@ def func(): print("func()") +def func2(): + print("func2()") + def return_func(): return func @@ -32,6 +35,9 @@ f4() # $ tt=func def return_func_in_dict_update(): d = {} d["func"] = func + d["func2"] = func2 + d["contested"] = func + d["contested"] = func2 return d dct2 = return_func_in_dict_update() # $ pt,tt=return_func_in_dict_update @@ -39,6 +45,10 @@ dct2 = return_func_in_dict_update() # $ pt,tt=return_func_in_dict_update f5 = dct2['func'] f5() # $ tt=func +dct2['func2']() # $ tt=func2 + +dct2['contested']() # $ tt=func2 SPURIOUS: tt=func + def return_func_in_list(): return [func, 42] From 92729dbbd659bdac0c8f9e31699161806fb80e73 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Fri, 23 Feb 2024 15:27:10 +0100 Subject: [PATCH 16/97] Python: Support iterable unpacking in type-tracking --- .../python/dataflow/new/internal/DataFlowPrivate.qll | 8 ++++---- .../library-tests/CallGraph/InlineCallGraphTest.expected | 1 - .../library-tests/CallGraph/code/func_ref_in_content.py | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll index 98841726a74..f2a52377544 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll @@ -650,6 +650,8 @@ predicate storeStepCommon(Node nodeFrom, ContentSet c, Node nodeTo) { dictStoreStep(nodeFrom, c, nodeTo) or moreDictStoreSteps(nodeFrom, c, nodeTo) + or + iterableUnpackingStoreStep(nodeFrom, c, nodeTo) } /** @@ -665,8 +667,6 @@ predicate storeStep(Node nodeFrom, ContentSet c, Node nodeTo) { or comprehensionStoreStep(nodeFrom, c, nodeTo) or - iterableUnpackingStoreStep(nodeFrom, c, nodeTo) - or attributeStoreStep(nodeFrom, c, nodeTo) or matchStoreStep(nodeFrom, c, nodeTo) @@ -903,6 +903,8 @@ predicate attributeStoreStep(Node nodeFrom, AttributeContent c, Node nodeTo) { */ predicate readStepCommon(Node nodeFrom, ContentSet c, Node nodeTo) { subscriptReadStep(nodeFrom, c, nodeTo) + or + iterableUnpackingReadStep(nodeFrom, c, nodeTo) } /** @@ -911,8 +913,6 @@ predicate readStepCommon(Node nodeFrom, ContentSet c, Node nodeTo) { predicate readStep(Node nodeFrom, ContentSet c, Node nodeTo) { readStepCommon(nodeFrom, c, nodeTo) or - iterableUnpackingReadStep(nodeFrom, c, nodeTo) - or matchReadStep(nodeFrom, c, nodeTo) or forReadStep(nodeFrom, c, nodeTo) diff --git a/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected b/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected index c5b7d6dc473..ef82a9ad20c 100644 --- a/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected +++ b/python/ql/test/experimental/library-tests/CallGraph/InlineCallGraphTest.expected @@ -15,7 +15,6 @@ pointsTo_found_typeTracker_notFound | code/func_defined_outside_class.py:39:11:39:21 | ControlFlowNode for _gen() | B._gen | | code/func_defined_outside_class.py:42:1:42:7 | ControlFlowNode for Attribute() | B._gen.func | | code/func_defined_outside_class.py:43:1:43:7 | ControlFlowNode for Attribute() | B._gen.func | -| code/func_ref_in_content.py:20:1:20:4 | ControlFlowNode for f2() | func | | code/funky_regression.py:15:9:15:17 | ControlFlowNode for Attribute() | Wat.f2 | | code/type_tracking_limitation.py:8:1:8:3 | ControlFlowNode for x() | my_func | typeTracker_found_pointsTo_notFound diff --git a/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py b/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py index b249ec0b2e9..24518ace088 100644 --- a/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py +++ b/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py @@ -17,7 +17,7 @@ def return_func_in_tuple(): tup = return_func_in_tuple() # $ pt,tt=return_func_in_tuple f2, _ = tup -f2() # $ pt=func MISSING: tt +f2() # $ pt,tt=func f3 = tup[0] f3() # $ tt,pt=func From 8a7ffac19c9162fdfbde38aa859fc114ad927ba6 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Fri, 1 Mar 2024 12:13:43 +0100 Subject: [PATCH 17/97] Python: Accept consistency failure --- .../CallGraph/CONSISTENCY/TypeTrackingConsistency.expected | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 python/ql/test/experimental/library-tests/CallGraph/CONSISTENCY/TypeTrackingConsistency.expected diff --git a/python/ql/test/experimental/library-tests/CallGraph/CONSISTENCY/TypeTrackingConsistency.expected b/python/ql/test/experimental/library-tests/CallGraph/CONSISTENCY/TypeTrackingConsistency.expected new file mode 100644 index 00000000000..6aed7c83813 --- /dev/null +++ b/python/ql/test/experimental/library-tests/CallGraph/CONSISTENCY/TypeTrackingConsistency.expected @@ -0,0 +1,2 @@ +| code/func_ref_in_content.py:19:1:19:5 | IterableElement | Unreachable node in step of kind store Tuple element at index 0. | +| code/func_ref_in_content.py:19:1:19:5 | IterableElement | Unreachable node in step of kind store Tuple element at index 1. | From 4d78762ba85ddb1373488ef1b0f45fc2a9f9b2a3 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Fri, 1 Mar 2024 12:14:14 +0100 Subject: [PATCH 18/97] Python: Ignore consistency failure --- python/ql/consistency-queries/TypeTrackingConsistency.ql | 7 +++++++ .../CallGraph/CONSISTENCY/TypeTrackingConsistency.expected | 2 -- 2 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 python/ql/test/experimental/library-tests/CallGraph/CONSISTENCY/TypeTrackingConsistency.expected diff --git a/python/ql/consistency-queries/TypeTrackingConsistency.ql b/python/ql/consistency-queries/TypeTrackingConsistency.ql index 15083229002..551573a7aef 100644 --- a/python/ql/consistency-queries/TypeTrackingConsistency.ql +++ b/python/ql/consistency-queries/TypeTrackingConsistency.ql @@ -36,6 +36,13 @@ private module ConsistencyChecksInput implements ConsistencyChecksInputSig { // which I couldn't just fix. We ignore the problems here, and instead rely on the // test-case added in https://github.com/github/codeql/pull/15841 n.getLocation().getFile().getAbsolutePath().matches("%/socketserver.py") + or + // for iterable unpacking like `a,b = some_list`, we currently don't want to allow + // type-tracking... however, in the future when we allow tracking list indexes + // precisely (that is, move away from ListElementContent), we should ensure we have + // proper flow to the synthetic `IterableElementNode`. + exists(DataFlow::ListElementContent c) and + n instanceof DataFlow::IterableElementNode } } diff --git a/python/ql/test/experimental/library-tests/CallGraph/CONSISTENCY/TypeTrackingConsistency.expected b/python/ql/test/experimental/library-tests/CallGraph/CONSISTENCY/TypeTrackingConsistency.expected deleted file mode 100644 index 6aed7c83813..00000000000 --- a/python/ql/test/experimental/library-tests/CallGraph/CONSISTENCY/TypeTrackingConsistency.expected +++ /dev/null @@ -1,2 +0,0 @@ -| code/func_ref_in_content.py:19:1:19:5 | IterableElement | Unreachable node in step of kind store Tuple element at index 0. | -| code/func_ref_in_content.py:19:1:19:5 | IterableElement | Unreachable node in step of kind store Tuple element at index 1. | From fa0c4e18fcbe2638af0266bbd86d209e2feec054 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Mon, 4 Mar 2024 16:05:18 +0100 Subject: [PATCH 19/97] Python: Expand dict-content tt test even more While it might be useful to track content to any lookup, it's not something we do right now. --- .../CallGraph/code/func_ref_in_content.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py b/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py index 24518ace088..eee8f29778b 100644 --- a/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py +++ b/python/ql/test/experimental/library-tests/CallGraph/code/func_ref_in_content.py @@ -50,6 +50,17 @@ dct2['func2']() # $ tt=func2 dct2['contested']() # $ tt=func2 SPURIOUS: tt=func +## non-precise access is not supported right now +for k in dct2: + dct2[k]() # $ MISSING: tt=func tt=func2 + +for v in dct2.values(): + v() # $ MISSING: tt=func tt=func2 + +for k, v in dct2.items(): + v() # $ MISSING: tt=func tt=func2 + + def return_func_in_list(): return [func, 42] From 7de304bf1680d61f7b6309cb840a941445f9caf9 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 12 Mar 2024 16:07:53 +0100 Subject: [PATCH 20/97] Python: Add proper type-tracking tests for content Instead of just relying on the call-graph tests --- .../dataflow/typetracking/content_test.py | 78 +++++++++++++++++++ .../dataflow/typetracking/tracked.ql | 8 ++ 2 files changed, 86 insertions(+) create mode 100644 python/ql/test/experimental/dataflow/typetracking/content_test.py diff --git a/python/ql/test/experimental/dataflow/typetracking/content_test.py b/python/ql/test/experimental/dataflow/typetracking/content_test.py new file mode 100644 index 00000000000..ac201f23301 --- /dev/null +++ b/python/ql/test/experimental/dataflow/typetracking/content_test.py @@ -0,0 +1,78 @@ +# test of other content types than attributes + +def test_tuple(index_arg): + tup = (tracked, other) # $tracked + + tup[0] # $ tracked + tup[1] + + a,b = tup # $tracked + a # $ tracked + b + + # non-precise access is not supported right now (and it's not 100% clear if we want + # to support it, or if it will lead to bad results) + tup[index_arg] + + for x in tup: + print(x) + + for i in range(len(tup)): + print(tup[i]) + + +def test_dict(key_arg): + d1 = {"t": tracked, "o": other} # $tracked + d1["t"] # $ tracked + d1.get("t") # $ MISSING: tracked + d1.setdefault("t") # $ MISSING: tracked + + d1["o"] + d1.get("o") + d1.setdefault("o") + + + # non-precise access is not supported right now (and it's not 100% clear if we want + # to support it, or if it will lead to bad results) + d1[key_arg] + + for k in d1: + d1[k] + + for v in d1.values(): + v + + for k, v in d1.items(): + v + + + # construction with inline updates + d2 = dict() + d2["t"] = tracked # $ tracked + d2["o"] = other + + d2["t"] # $ tracked + d2["o"] + + # notice that time-travel is also possible (just as with attributes) + d3 = dict() + d3["t"] # $ SPURIOUS: tracked + d3["t"] = tracked # $ tracked + d3["t"] # $ tracked + + +def test_list(index_arg): + l = [tracked, other] # $tracked + + l[0] # $ MISSING: tracked + l[1] + + # non-precise access is not supported right now (and it's not 100% clear if we want + # to support it, or if it will lead to bad results) + l[index_arg] + + for x in l: + print(x) + + for i in range(len(l)): + print(l[i]) diff --git a/python/ql/test/experimental/dataflow/typetracking/tracked.ql b/python/ql/test/experimental/dataflow/typetracking/tracked.ql index ca893688256..8bad0e33ead 100644 --- a/python/ql/test/experimental/dataflow/typetracking/tracked.ql +++ b/python/ql/test/experimental/dataflow/typetracking/tracked.ql @@ -30,6 +30,14 @@ module TrackedTest implements TestSig { not e instanceof DataFlow::ScopeEntryDefinitionNode and // ...same for `SynthCaptureNode`s not e instanceof DP::SynthCaptureNode and + // after starting to track all kinds of content, we generally just want to show + // annotations after reading the tracked data out again. (we keep the old + // attribute logic to not rewrite all our tests) + ( + t.getContent().isNone() + or + t.getContent().asSome() instanceof DataFlow::AttributeContent + ) and tag = "tracked" and location = e.getLocation() and value = t.getAttr() and From 2b09b084e0393193e5daf97a1a512538c9aa4b57 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 12 Mar 2024 17:43:43 +0100 Subject: [PATCH 21/97] Python: Add change-note --- python/ql/lib/change-notes/2024-03-12-typetracking-content.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 python/ql/lib/change-notes/2024-03-12-typetracking-content.md diff --git a/python/ql/lib/change-notes/2024-03-12-typetracking-content.md b/python/ql/lib/change-notes/2024-03-12-typetracking-content.md new file mode 100644 index 00000000000..5ad93a657ae --- /dev/null +++ b/python/ql/lib/change-notes/2024-03-12-typetracking-content.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* Improved the type-tracking capabilities (and therefore also API graphs) to allow tracking items in tuples and dictionaries. From af8cef5b535b068057a58654701cf27ca2641e81 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 12 Mar 2024 17:57:32 +0100 Subject: [PATCH 22/97] Python: Fixup deprecated type-tracker API --- .../dataflow/new/internal/TypeTracker.qll | 46 +++++++++++++++---- .../new/internal/TypeTrackerSpecific.qll | 2 +- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/TypeTracker.qll b/python/ql/lib/semmle/python/dataflow/new/internal/TypeTracker.qll index 0f6ff8bd3bd..01c881b2316 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/TypeTracker.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/TypeTracker.qll @@ -1,6 +1,7 @@ /** Step Summaries and Type Tracking */ private import TypeTrackerSpecific +private import semmle.python.dataflow.new.internal.DataFlowPublic as DataFlowPublic cached private module Cached { @@ -12,10 +13,22 @@ private module Cached { LevelStep() or CallStep() or ReturnStep() or - deprecated StoreStep(TypeTrackerContent content) { basicStoreStep(_, _, content) } or - deprecated LoadStep(TypeTrackerContent content) { basicLoadStep(_, _, content) } or + deprecated StoreStep(TypeTrackerContent content) { + exists(DataFlowPublic::AttributeContent dfc | dfc.getAttribute() = content | + basicStoreStep(_, _, dfc) + ) + } or + deprecated LoadStep(TypeTrackerContent content) { + exists(DataFlowPublic::AttributeContent dfc | dfc.getAttribute() = content | + basicLoadStep(_, _, dfc) + ) + } or deprecated LoadStoreStep(TypeTrackerContent load, TypeTrackerContent store) { - basicLoadStoreStep(_, _, load, store) + exists(DataFlowPublic::AttributeContent dfcLoad, DataFlowPublic::AttributeContent dfcStore | + dfcLoad.getAttribute() = load and dfcStore.getAttribute() = store + | + basicLoadStoreStep(_, _, dfcLoad, dfcStore) + ) } or deprecated WithContent(ContentFilter filter) { basicWithContentStep(_, _, filter) } or deprecated WithoutContent(ContentFilter filter) { basicWithoutContentStep(_, _, filter) } or @@ -29,13 +42,13 @@ private module Cached { // Restrict `content` to those that might eventually match a load. // We can't rely on `basicStoreStep` since `startInContent` might be used with // a content that has no corresponding store. - exists(TypeTrackerContent loadContents | + exists(DataFlowPublic::AttributeContent loadContents | ( basicLoadStep(_, _, loadContents) or basicLoadStoreStep(_, _, loadContents, _) ) and - compatibleContents(content, loadContents) + compatibleContents(content, loadContents.getAttribute()) ) } @@ -45,13 +58,13 @@ private module Cached { content = noContent() or // As in MkTypeTracker, restrict `content` to those that might eventually match a store. - exists(TypeTrackerContent storeContent | + exists(DataFlowPublic::AttributeContent storeContent | ( basicStoreStep(_, _, storeContent) or basicLoadStoreStep(_, _, _, storeContent) ) and - compatibleContents(storeContent, content) + compatibleContents(storeContent.getAttribute(), content) ) } @@ -198,7 +211,10 @@ private module Cached { flowsToStoreStep(nodeFrom, nodeTo, content) and summary = StoreStep(content) or - basicLoadStep(nodeFrom, nodeTo, content) and summary = LoadStep(content) + exists(DataFlowPublic::AttributeContent dfc | dfc.getAttribute() = content | + basicLoadStep(nodeFrom, nodeTo, dfc) + ) and + summary = LoadStep(content) ) or exists(TypeTrackerContent loadContent, TypeTrackerContent storeContent | @@ -281,7 +297,12 @@ deprecated private predicate smallstepProj(Node nodeFrom, StepSummary summary) { deprecated private predicate flowsToStoreStep( Node nodeFrom, TypeTrackingNode nodeTo, TypeTrackerContent content ) { - exists(Node obj | nodeTo.flowsTo(obj) and basicStoreStep(nodeFrom, obj, content)) + exists(Node obj | + nodeTo.flowsTo(obj) and + exists(DataFlowPublic::AttributeContent dfc | dfc.getAttribute() = content | + basicStoreStep(nodeFrom, obj, dfc) + ) + ) } /** @@ -292,7 +313,12 @@ deprecated private predicate flowsToLoadStoreStep( TypeTrackerContent storeContent ) { exists(Node obj | - nodeTo.flowsTo(obj) and basicLoadStoreStep(nodeFrom, obj, loadContent, storeContent) + nodeTo.flowsTo(obj) and + exists(DataFlowPublic::AttributeContent loadDfc, DataFlowPublic::AttributeContent storeDfc | + loadDfc.getAttribute() = loadContent and storeDfc.getAttribute() = storeContent + | + basicLoadStoreStep(nodeFrom, obj, loadDfc, storeDfc) + ) ) } diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackerSpecific.qll b/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackerSpecific.qll index c31cfeb5331..11cce1446f7 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackerSpecific.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackerSpecific.qll @@ -15,7 +15,7 @@ deprecated class OptionalTypeTrackerContent extends string { OptionalTypeTrackerContent() { this = "" or - this instanceof TypeTrackingImpl::TypeTrackingInput::Content + this = any(DataFlowPublic::AttributeContent dfc).getAttribute() } } From 6ffaad1bc8cfb0812549780d6547c8e37a2dcea9 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 12 Mar 2024 22:32:40 +0100 Subject: [PATCH 23/97] Python: Expand type-tracking tests with nested tuples I was initially surprised to see that this didn't work, until I remembered that type-tracking only works with content of depth 1. --- .../TypeTrackingConsistency.expected | 13 +++++++++++ .../dataflow/typetracking/content_test.py | 22 +++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 python/ql/test/experimental/dataflow/typetracking/CONSISTENCY/TypeTrackingConsistency.expected diff --git a/python/ql/test/experimental/dataflow/typetracking/CONSISTENCY/TypeTrackingConsistency.expected b/python/ql/test/experimental/dataflow/typetracking/CONSISTENCY/TypeTrackingConsistency.expected new file mode 100644 index 00000000000..6fc4df91699 --- /dev/null +++ b/python/ql/test/experimental/dataflow/typetracking/CONSISTENCY/TypeTrackingConsistency.expected @@ -0,0 +1,13 @@ +unreachableNode +| content_test.py:31:6:31:11 | ControlFlowNode for Tuple | Unreachable node in step of kind load Tuple element at index 0. | +| content_test.py:31:6:31:11 | ControlFlowNode for Tuple | Unreachable node in step of kind load Tuple element at index 1. | +| content_test.py:31:6:31:11 | ControlFlowNode for Tuple | Unreachable node in step of kind storeTarget. | +| content_test.py:31:16:31:21 | ControlFlowNode for Tuple | Unreachable node in step of kind load Tuple element at index 0. | +| content_test.py:31:16:31:21 | ControlFlowNode for Tuple | Unreachable node in step of kind load Tuple element at index 1. | +| content_test.py:31:16:31:21 | ControlFlowNode for Tuple | Unreachable node in step of kind storeTarget. | +| content_test.py:40:10:40:13 | ControlFlowNode for Tuple | Unreachable node in step of kind load Tuple element at index 0. | +| content_test.py:40:10:40:13 | ControlFlowNode for Tuple | Unreachable node in step of kind load Tuple element at index 1. | +| content_test.py:40:10:40:13 | ControlFlowNode for Tuple | Unreachable node in step of kind storeTarget. | +| content_test.py:66:9:66:12 | ControlFlowNode for Tuple | Unreachable node in step of kind load Tuple element at index 0. | +| content_test.py:66:9:66:12 | ControlFlowNode for Tuple | Unreachable node in step of kind load Tuple element at index 1. | +| content_test.py:66:9:66:12 | ControlFlowNode for Tuple | Unreachable node in step of kind storeTarget. | diff --git a/python/ql/test/experimental/dataflow/typetracking/content_test.py b/python/ql/test/experimental/dataflow/typetracking/content_test.py index ac201f23301..1c52d659582 100644 --- a/python/ql/test/experimental/dataflow/typetracking/content_test.py +++ b/python/ql/test/experimental/dataflow/typetracking/content_test.py @@ -21,6 +21,28 @@ def test_tuple(index_arg): print(tup[i]) + # nested tuples + nested_tuples = ((tracked, other), (other, tracked)) # $tracked + + nested_tuples[0][0] # $ MISSING: tracked + nested_tuples[0][1] + nested_tuples[1][0] + nested_tuples[1][1] # $ MISSING: tracked + + (aa, ab), (ba, bb) = nested_tuples + aa # $ MISSING: tracked + ab + ba + bb # $ MISSING: tracked + + + # non-precise access is not supported right now (and it's not 100% clear if we want + # to support it, or if it will lead to bad results) + for (x, y) in nested_tuples: + x + y + + def test_dict(key_arg): d1 = {"t": tracked, "o": other} # $tracked d1["t"] # $ tracked From 7a3ee0f5f8145abe25b26a64415a43ed32b7016e Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 13 Mar 2024 16:41:42 +0100 Subject: [PATCH 24/97] Python: Make `IterableSequenceNode` LocalSourceNode We do this to remove the inconsistencies, and to be ready for a future where type-tracking support content tracker of depth > 1. It works because targets of loadSteps needs to be LocalSourceNodes predicate loadStep(Node nodeFrom, LocalSourceNode nodeTo, Content content) { --- .../python/dataflow/new/internal/LocalSources.qll | 2 ++ .../CONSISTENCY/TypeTrackingConsistency.expected | 13 ------------- 2 files changed, 2 insertions(+), 13 deletions(-) delete mode 100644 python/ql/test/experimental/dataflow/typetracking/CONSISTENCY/TypeTrackingConsistency.expected diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/LocalSources.qll b/python/ql/lib/semmle/python/dataflow/new/internal/LocalSources.qll index 34b137b3511..92d9e5887ad 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/LocalSources.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/LocalSources.qll @@ -74,6 +74,8 @@ class LocalSourceNode extends Node { this instanceof ScopeEntryDefinitionNode or this instanceof ParameterNode + or + this instanceof IterableSequenceNode } /** Holds if this `LocalSourceNode` can flow to `nodeTo` in one or more local flow steps. */ diff --git a/python/ql/test/experimental/dataflow/typetracking/CONSISTENCY/TypeTrackingConsistency.expected b/python/ql/test/experimental/dataflow/typetracking/CONSISTENCY/TypeTrackingConsistency.expected deleted file mode 100644 index 6fc4df91699..00000000000 --- a/python/ql/test/experimental/dataflow/typetracking/CONSISTENCY/TypeTrackingConsistency.expected +++ /dev/null @@ -1,13 +0,0 @@ -unreachableNode -| content_test.py:31:6:31:11 | ControlFlowNode for Tuple | Unreachable node in step of kind load Tuple element at index 0. | -| content_test.py:31:6:31:11 | ControlFlowNode for Tuple | Unreachable node in step of kind load Tuple element at index 1. | -| content_test.py:31:6:31:11 | ControlFlowNode for Tuple | Unreachable node in step of kind storeTarget. | -| content_test.py:31:16:31:21 | ControlFlowNode for Tuple | Unreachable node in step of kind load Tuple element at index 0. | -| content_test.py:31:16:31:21 | ControlFlowNode for Tuple | Unreachable node in step of kind load Tuple element at index 1. | -| content_test.py:31:16:31:21 | ControlFlowNode for Tuple | Unreachable node in step of kind storeTarget. | -| content_test.py:40:10:40:13 | ControlFlowNode for Tuple | Unreachable node in step of kind load Tuple element at index 0. | -| content_test.py:40:10:40:13 | ControlFlowNode for Tuple | Unreachable node in step of kind load Tuple element at index 1. | -| content_test.py:40:10:40:13 | ControlFlowNode for Tuple | Unreachable node in step of kind storeTarget. | -| content_test.py:66:9:66:12 | ControlFlowNode for Tuple | Unreachable node in step of kind load Tuple element at index 0. | -| content_test.py:66:9:66:12 | ControlFlowNode for Tuple | Unreachable node in step of kind load Tuple element at index 1. | -| content_test.py:66:9:66:12 | ControlFlowNode for Tuple | Unreachable node in step of kind storeTarget. | From 00f2a6a65e76336e0cd926a1630dd0001ed3bd14 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Fri, 15 Mar 2024 10:14:45 +0100 Subject: [PATCH 25/97] Python: Update ssa-compute test expectations --- .../CONSISTENCY/TypeTrackingConsistency.expected | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/ql/test/library-tests/essa/ssa-compute/CONSISTENCY/TypeTrackingConsistency.expected b/python/ql/test/library-tests/essa/ssa-compute/CONSISTENCY/TypeTrackingConsistency.expected index 81d19f3f20d..0e829fd207f 100644 --- a/python/ql/test/library-tests/essa/ssa-compute/CONSISTENCY/TypeTrackingConsistency.expected +++ b/python/ql/test/library-tests/essa/ssa-compute/CONSISTENCY/TypeTrackingConsistency.expected @@ -1,6 +1,6 @@ unreachableNode -| test2.py:16:17:16:17 | ControlFlowNode for y | Unreachable node in step of kind load bar. | -| test2.py:25:23:25:23 | ControlFlowNode for x | Unreachable node in step of kind load attribute. | +| test2.py:16:17:16:17 | ControlFlowNode for y | Unreachable node in step of kind load Attribute bar. | +| test2.py:25:23:25:23 | ControlFlowNode for x | Unreachable node in step of kind load Attribute attribute. | | test2.py:25:23:25:23 | ControlFlowNode for x | Unreachable node in step of kind simpleLocalSmallStep. | -| test2.py:26:17:26:17 | ControlFlowNode for y | Unreachable node in step of kind load bar. | +| test2.py:26:17:26:17 | ControlFlowNode for y | Unreachable node in step of kind load Attribute bar. | | test2.py:27:23:27:23 | ControlFlowNode for x | Unreachable node in step of kind simpleLocalSmallStep. | From 6babb2ff909d750c908ed3f2faadc8a5285fe81e Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Fri, 15 Mar 2024 10:24:33 +0100 Subject: [PATCH 26/97] Python: Accept .expected for `typetracking-summaries` --- .../experimental/dataflow/typetracking-summaries/summaries.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ql/test/experimental/dataflow/typetracking-summaries/summaries.py b/python/ql/test/experimental/dataflow/typetracking-summaries/summaries.py index e11f451b865..89b5e1756d5 100644 --- a/python/ql/test/experimental/dataflow/typetracking-summaries/summaries.py +++ b/python/ql/test/experimental/dataflow/typetracking-summaries/summaries.py @@ -41,8 +41,8 @@ tms = tainted_mapped_summary[0] tms # $ MISSING: tracked another_tainted_list = TTS_append_to_list([], tracked) # $ tracked -atl = another_tainted_list[0] -atl # $ MISSING: tracked +atl = another_tainted_list[0] # $ tracked +atl # $ tracked # This will not work, as the call is not found by `getACallSimple`. from json import loads as json_loads From 7eb4419342e08b9f537cd75e1db2e0f5693c55fe Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Fri, 15 Mar 2024 10:24:57 +0100 Subject: [PATCH 27/97] Python: Restrict type-tracking content to only be precise At least for now :) --- .../dataflow/new/internal/TypeTrackingImpl.qll | 16 +++++++++++++++- .../dataflow/typetracking-summaries/summaries.py | 4 ++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll b/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll index 8b3e1a95ef1..ce95a6cca4e 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll @@ -102,7 +102,21 @@ module TypeTrackingInput implements Shared::TypeTrackingInput { class LocalSourceNode = DataFlowPublic::LocalSourceNode; - class Content = DataFlowPublic::Content; + class Content extends DataFlowPublic::Content { + Content() { + // TODO: for now, it's not 100% clear if should support non-precise content in + // type-tracking, or if it will lead to bad results. We start with only allowing + // precise content, which should always be a good improvement! It also simplifies + // the process of examining new results from non-precise content steps in the + // future, since you will _only_ have to look over the results from the new + // non-precise steps. + this instanceof DataFlowPublic::AttributeContent + or + this instanceof DataFlowPublic::DictionaryElementContent + or + this instanceof DataFlowPublic::TupleElementContent + } + } /** * A label to use for `WithContent` and `WithoutContent` steps, restricting diff --git a/python/ql/test/experimental/dataflow/typetracking-summaries/summaries.py b/python/ql/test/experimental/dataflow/typetracking-summaries/summaries.py index 89b5e1756d5..e11f451b865 100644 --- a/python/ql/test/experimental/dataflow/typetracking-summaries/summaries.py +++ b/python/ql/test/experimental/dataflow/typetracking-summaries/summaries.py @@ -41,8 +41,8 @@ tms = tainted_mapped_summary[0] tms # $ MISSING: tracked another_tainted_list = TTS_append_to_list([], tracked) # $ tracked -atl = another_tainted_list[0] # $ tracked -atl # $ tracked +atl = another_tainted_list[0] +atl # $ MISSING: tracked # This will not work, as the call is not found by `getACallSimple`. from json import loads as json_loads From 32ebd4eebbcf13613b51c4b8686b3183b0da3c04 Mon Sep 17 00:00:00 2001 From: Max Schaefer Date: Wed, 27 Mar 2024 12:22:24 +0000 Subject: [PATCH 28/97] Automodel: Filter unexploitable types in application mode. We already did this in framework mode. --- .../AutomodelApplicationModeCharacteristics.qll | 16 +++++++++------- .../AutomodelApplicationModeExtraction/Test.java | 11 ++++++----- .../com/github/codeql/test/MyWriter.java | 2 +- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/java/ql/automodel/src/AutomodelApplicationModeCharacteristics.qll b/java/ql/automodel/src/AutomodelApplicationModeCharacteristics.qll index 045886985b8..3023677ece8 100644 --- a/java/ql/automodel/src/AutomodelApplicationModeCharacteristics.qll +++ b/java/ql/automodel/src/AutomodelApplicationModeCharacteristics.qll @@ -28,25 +28,27 @@ newtype TApplicationModeEndpoint = AutomodelJavaUtil::isFromSource(call) and exists(Argument argExpr | arg.asExpr() = argExpr and call = argExpr.getCall() and not argExpr.isVararg() - ) + ) and + not AutomodelJavaUtil::isUnexploitableType(arg.getType()) } or TInstanceArgument(Call call, DataFlow::Node arg) { AutomodelJavaUtil::isFromSource(call) and arg = DataFlow::getInstanceArgument(call) and - not call instanceof ConstructorCall + not call instanceof ConstructorCall and + not AutomodelJavaUtil::isUnexploitableType(arg.getType()) } or TImplicitVarargsArray(Call call, DataFlow::ImplicitVarargsArray arg, int idx) { AutomodelJavaUtil::isFromSource(call) and call = arg.getCall() and - idx = call.getCallee().getVaragsParameterIndex() + idx = call.getCallee().getVaragsParameterIndex() and + not AutomodelJavaUtil::isUnexploitableType(arg.getType()) } or - TMethodReturnValue(Call call) { + TMethodReturnValue(MethodCall call) { AutomodelJavaUtil::isFromSource(call) and - not call instanceof ConstructorCall + not AutomodelJavaUtil::isUnexploitableType(call.getType()) } or TOverriddenParameter(Parameter p, Method overriddenMethod) { AutomodelJavaUtil::isFromSource(p) and - not p.getCallable().callsConstructor(_) and p.getCallable().(Method).overrides(overriddenMethod) } @@ -163,7 +165,7 @@ class ImplicitVarargsArray extends CallArgument, TImplicitVarargsArray { * may be a source. */ class MethodReturnValue extends ApplicationModeEndpoint, TMethodReturnValue { - Call call; + MethodCall call; MethodReturnValue() { this = TMethodReturnValue(call) } diff --git a/java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java b/java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java index 4d6aff63fd0..a8280bcaf27 100644 --- a/java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java +++ b/java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java @@ -19,11 +19,11 @@ class Test { AtomicReference reference = new AtomicReference<>(); // uninteresting (parameterless constructor) reference.set( // $ sinkModelCandidate=set(Object):Argument[this] args[0] // $ negativeSinkExample=set(Object):Argument[0] // modeled as a flow step - ); // $ negativeSourceExample=set(Object):ReturnValue // return type is void + ); // not a source candidate (return type is void) } public static void callSupplier(Supplier supplier) { - supplier.get(); // $ sourceModelCandidate=get():ReturnValue + supplier.get(); // not a source candidate (lambda flow) } public static void copyFiles(Path source, Path target, CopyOption option) throws Exception { @@ -52,7 +52,7 @@ class Test { public static int compareFiles(File f1, File f2) { return f1.compareTo( // $ negativeSinkExample=compareTo(File):Argument[this] f2 // $ negativeSinkExample=compareTo(File):Argument[0] // modeled as not a sink - ); // $ negativeSourceExample=compareTo(File):ReturnValue // return type is int + ); // not a source candidate (return type is int) } public static void FilesWalkExample(Path p, FileVisitOption o) throws Exception { @@ -66,6 +66,7 @@ class Test { public static void WebSocketExample(URLConnection c) throws Exception { c.getInputStream(); // $ sinkModelCandidate=getInputStream():Argument[this] positiveSourceExample=getInputStream():ReturnValue(remote) // not a source candidate (manual modeling) + c.connect(); // $ sinkModelCandidate=connect():Argument[this] // not a source candidate (return type is void) } public static void fileFilterExample(File f, FileFilter ff) { @@ -102,10 +103,10 @@ class MoreTests { Files.delete( p // $ sinkModelCandidate=delete(Path):Argument[0] positiveSinkExample=delete(Path):Argument[0](path-injection) - ); // $ negativeSourceExample=delete(Path):ReturnValue // return type is void + ); // $ not a source candidate (return type is void) Files.deleteIfExists( p // $ sinkModelCandidate=deleteIfExists(Path):Argument[0] positiveSinkExample=deleteIfExists(Path):Argument[0](path-injection) - ); // $ negativeSourceExample=deleteIfExists(Path):ReturnValue // return type is boolean + ); // $ not a source candidate (return type is boolean) } } \ No newline at end of file diff --git a/java/ql/automodel/test/AutomodelFrameworkModeExtraction/com/github/codeql/test/MyWriter.java b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/com/github/codeql/test/MyWriter.java index b31ace21b4d..62bd773cc2e 100644 --- a/java/ql/automodel/test/AutomodelFrameworkModeExtraction/com/github/codeql/test/MyWriter.java +++ b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/com/github/codeql/test/MyWriter.java @@ -2,7 +2,7 @@ package com.github.codeql.test; public class MyWriter extends java.io.Writer { @Override - public void write(char[] cbuf, int off, int len) { // $ sinkModelCandidate=write(char[],int,int):Argument[this] sourceModelCandidate=write(char[],int,int):Parameter[this] sourceModelCandidate=write(char[],int,int):Parameter[0] + public void write(char[] cbuf, int off, int len) { // $ sinkModelCandidate=write(char[],int,int):Argument[this] positiveSinkExample=write(char[],int,int):Argument[0](file-content-store) sourceModelCandidate=write(char[],int,int):Parameter[this] sourceModelCandidate=write(char[],int,int):Parameter[0] } @Override From 20202aba908a7d0a3e6fc3aa64d2991e838e8768 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 2 Apr 2024 13:21:46 +0200 Subject: [PATCH 29/97] Python: Deprecate `AttributeName` --- python/ql/lib/semmle/python/dataflow/new/TypeTracking.qll | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/ql/lib/semmle/python/dataflow/new/TypeTracking.qll b/python/ql/lib/semmle/python/dataflow/new/TypeTracking.qll index 9d0bcb3c487..8d1c691915b 100644 --- a/python/ql/lib/semmle/python/dataflow/new/TypeTracking.qll +++ b/python/ql/lib/semmle/python/dataflow/new/TypeTracking.qll @@ -7,8 +7,12 @@ private import internal.TypeTrackingImpl as Impl import Impl::Shared::TypeTracking private import semmle.python.dataflow.new.internal.DataFlowPublic as DataFlowPublic -/** A string that may appear as the name of an attribute or access path. */ -class AttributeName = Impl::TypeTrackingInput::Content; +/** + * DEPRECATED. + * + * A string that may appear as the name of an attribute or access path. + */ +deprecated class AttributeName = Impl::TypeTrackingInput::Content; /** * A summary of the steps needed to track a value to a given dataflow node. From 8707a63edb9440890234bc4ab0019c099087f56f Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 2 Apr 2024 13:26:26 +0200 Subject: [PATCH 30/97] Python: Add comments around `storeStepCommon` --- .../python/dataflow/new/internal/DataFlowPrivate.qll | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll index f2a52377544..1ad6d0f7e6e 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll @@ -643,6 +643,13 @@ predicate jumpStepNotSharedWithTypeTracker(Node nodeFrom, Node nodeTo) { //-------- /** * Subset of `storeStep` that should be shared with type-tracking. + * + * NOTE: This does not include attributeStoreStep right now, since it has its' own + * modeling in the type-tracking library (which is slightly different due to + * PostUpdateNodes). + * + * As of 2024-04-02 the type-tracking library only supports precise content, so there is + * no reason to include steps for list content right now. */ predicate storeStepCommon(Node nodeFrom, ContentSet c, Node nodeTo) { tupleStoreStep(nodeFrom, c, nodeTo) From a22b9947c02eee14b05205ab7c520a2add13de5b Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 2 Apr 2024 14:52:36 +0200 Subject: [PATCH 31/97] Python: Revert `IterableSequenceNode` as LocalSourceNode When looking things over a bit more, we could actually exclude the steps that would never be used instead. A much more involved solution, but more performance oriented and clear in terms of what is supported (at least until we start supporting type-tracking with more than depth 1 access-path, if that ever happens) --- .../TypeTrackingConsistency.ql | 4 -- .../dataflow/new/internal/LocalSources.qll | 2 - .../new/internal/TypeTrackingImpl.qll | 45 +++++++++++++++++-- 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/python/ql/consistency-queries/TypeTrackingConsistency.ql b/python/ql/consistency-queries/TypeTrackingConsistency.ql index 551573a7aef..645bdef5219 100644 --- a/python/ql/consistency-queries/TypeTrackingConsistency.ql +++ b/python/ql/consistency-queries/TypeTrackingConsistency.ql @@ -27,10 +27,6 @@ private module ConsistencyChecksInput implements ConsistencyChecksInputSig { TypeTrackingInput::simpleLocalSmallStep*(m, n) ) or - // TODO: when adding support for proper content, handle iterable unpacking better - // such as `for k,v in items:`, or `a, (b,c) = ...` - n instanceof DataFlow::IterableSequenceNode - or // We have missing use-use flow in // https://github.com/python/cpython/blob/0fb18b02c8ad56299d6a2910be0bab8ad601ef24/Lib/socketserver.py#L276-L303 // which I couldn't just fix. We ignore the problems here, and instead rely on the diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/LocalSources.qll b/python/ql/lib/semmle/python/dataflow/new/internal/LocalSources.qll index 92d9e5887ad..34b137b3511 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/LocalSources.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/LocalSources.qll @@ -74,8 +74,6 @@ class LocalSourceNode extends Node { this instanceof ScopeEntryDefinitionNode or this instanceof ParameterNode - or - this instanceof IterableSequenceNode } /** Holds if this `LocalSourceNode` can flow to `nodeTo` in one or more local flow steps. */ diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll b/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll index ce95a6cca4e..42ce5cdd237 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackingImpl.qll @@ -8,6 +8,7 @@ private import semmle.python.dataflow.new.internal.DataFlowPrivate as DataFlowPr private import codeql.typetracking.internal.SummaryTypeTracker as SummaryTypeTracker private import semmle.python.dataflow.new.internal.FlowSummaryImpl as FlowSummaryImpl private import semmle.python.dataflow.new.internal.DataFlowDispatch as DataFlowDispatch +private import semmle.python.dataflow.new.internal.IterableUnpacking as IterableUnpacking private module SummaryTypeTrackerInput implements SummaryTypeTracker::Input { // Dataflow nodes @@ -135,7 +136,27 @@ module TypeTrackingInput implements Shared::TypeTrackingInput { } /** Holds if there is a simple local flow step from `nodeFrom` to `nodeTo` */ - predicate simpleLocalSmallStep = DataFlowPrivate::simpleLocalFlowStepForTypetracking/2; + predicate simpleLocalSmallStep(Node nodeFrom, Node nodeTo) { + DataFlowPrivate::simpleLocalFlowStepForTypetracking(nodeFrom, nodeTo) and + // for `for k,v in foo` no need to do local flow step from the synthetic sequence + // node for `k,v` to the tuple `k,v` -- since type-tracking only supports one level + // of content tracking, and there is one read-step from `foo` the synthetic sequence + // node required, we can skip the flow step from the synthetic sequence node to the + // tuple itself, since the read-step from the tuple to the tuple elements will not + // matter. + not ( + IterableUnpacking::iterableUnpackingForReadStep(_, _, nodeFrom) and + IterableUnpacking::iterableUnpackingTupleFlowStep(nodeFrom, nodeTo) + ) and + // for nested iterable unpacking, such as `[[a]] = foo` or `((a,b),) = bar`, we can + // ignore the flow steps from the synthetic sequence node to the real sequence node, + // since we only support one level of content in type-trackers, and the nested + // structure requires two levels at least to be useful. + not exists(SequenceNode outer | + outer.getAnElement() = nodeTo.asCfgNode() and + IterableUnpacking::iterableUnpackingTupleFlowStep(nodeFrom, nodeTo) + ) + } /** Holds if there is a level step from `nodeFrom` to `nodeTo`, which may depend on the call graph. */ predicate levelStepCall(Node nodeFrom, LocalSourceNode nodeTo) { none() } @@ -200,7 +221,10 @@ module TypeTrackingInput implements Shared::TypeTrackingInput { nodeTo = storeTarget or nodeTo = storeTarget.(DataFlowPrivate::SyntheticPostUpdateNode).getPreUpdateNode() - ) + ) and + // when only supporting precise content, no need for IterableElementNode (since it + // is only fed set/list content) + not nodeFrom instanceof DataFlowPublic::IterableElementNode or TypeTrackerSummaryFlow::basicStoreStep(nodeFrom, nodeTo, content) } @@ -216,7 +240,22 @@ module TypeTrackingInput implements Shared::TypeTrackingInput { nodeTo = a ) or - DataFlowPrivate::readStepCommon(nodeFrom, content, nodeTo) + DataFlowPrivate::readStepCommon(nodeFrom, content, nodeTo) and + // Since we only support one level of content in type-trackers we don't actually + // support `(aa, ab), (ba, bb) = ...`. Therefore we exclude the read-step from `(aa, + // ab)` to `aa` (since it is not needed). + not exists(SequenceNode outer | + outer.getAnElement() = nodeFrom.asCfgNode() and + IterableUnpacking::iterableUnpackingTupleFlowStep(_, nodeFrom) + ) and + // Again, due to only supporting one level deep, for `for (k,v) in ...` we exclude read-step from + // the tuple to `k` and `v`. + not exists(DataFlowPublic::IterableSequenceNode seq, DataFlowPublic::IterableElementNode elem | + IterableUnpacking::iterableUnpackingForReadStep(_, _, seq) and + IterableUnpacking::iterableUnpackingConvertingReadStep(seq, _, elem) and + IterableUnpacking::iterableUnpackingConvertingStoreStep(elem, _, nodeFrom) and + nodeFrom.asCfgNode() instanceof SequenceNode + ) or TypeTrackerSummaryFlow::basicLoadStep(nodeFrom, nodeTo, content) } From ce98353d2267558ab05574d2e2c4e2f773e973c8 Mon Sep 17 00:00:00 2001 From: Dave Bartolomeo Date: Tue, 2 Apr 2024 15:15:11 -0400 Subject: [PATCH 32/97] Allow `@param` tags to apply to record parameters --- .../Advisory/Documentation/SpuriousJavadocParam.ql | 12 ++++++++++++ .../src/change-notes/2024-04-02-javadoc-records.md | 5 +++++ .../test/query-tests/SpuriousJavadocParam/Test.java | 13 +++++++++++++ .../test/query-tests/SpuriousJavadocParam/options | 1 + .../query-tests/SpuriousJavadocParam/test.expected | 2 ++ 5 files changed, 33 insertions(+) create mode 100644 java/ql/src/change-notes/2024-04-02-javadoc-records.md create mode 100644 java/ql/test/query-tests/SpuriousJavadocParam/options diff --git a/java/ql/src/Advisory/Documentation/SpuriousJavadocParam.ql b/java/ql/src/Advisory/Documentation/SpuriousJavadocParam.ql index 691811687a7..43afd978f01 100644 --- a/java/ql/src/Advisory/Documentation/SpuriousJavadocParam.ql +++ b/java/ql/src/Advisory/Documentation/SpuriousJavadocParam.ql @@ -32,12 +32,24 @@ where ) or documentable instanceof ClassOrInterface and + not documentable instanceof Record and not exists(TypeVariable tv | tv.getGenericType() = documentable | "<" + tv.getName() + ">" = paramTag.getParamName() ) and msg = "@param tag \"" + paramTag.getParamName() + "\" does not match any actual type parameter of type \"" + documentable.getName() + "\"." + or + documentable instanceof Record and + not exists(TypeVariable tv | tv.getGenericType() = documentable | + "<" + tv.getName() + ">" = paramTag.getParamName() + ) and + not documentable.(Record).getCanonicalConstructor().getAParameter().getName() = + paramTag.getParamName() and + msg = + "@param tag \"" + paramTag.getParamName() + + "\" does not match any actual type parameter or record parameter of record \"" + + documentable.getName() + "\"." else // The tag has no value at all. msg = "This @param tag does not have a value." diff --git a/java/ql/src/change-notes/2024-04-02-javadoc-records.md b/java/ql/src/change-notes/2024-04-02-javadoc-records.md new file mode 100644 index 00000000000..e3859c9618b --- /dev/null +++ b/java/ql/src/change-notes/2024-04-02-javadoc-records.md @@ -0,0 +1,5 @@ +--- +category: minorAnalysis +--- +* The `java/unknown-javadoc-parameter` now accepts `@param` tags that apply to the parameters of a + record. diff --git a/java/ql/test/query-tests/SpuriousJavadocParam/Test.java b/java/ql/test/query-tests/SpuriousJavadocParam/Test.java index 84e05540c05..de108008ee8 100644 --- a/java/ql/test/query-tests/SpuriousJavadocParam/Test.java +++ b/java/ql/test/query-tests/SpuriousJavadocParam/Test.java @@ -120,5 +120,18 @@ public class Test { */ interface GenericInterface {} + /** + * @param i exists + * @param k does not + */ + static record SomeRecord(int i, int j) {} + + /** + * @param exists + * @param i exists + * @param k does not + */ + static record GenericRecord(int i, int j) {} + // Diagnostic Matches: Incomplete inheritance relation for type java.lang.Object and supertype none } diff --git a/java/ql/test/query-tests/SpuriousJavadocParam/options b/java/ql/test/query-tests/SpuriousJavadocParam/options new file mode 100644 index 00000000000..fc57fe025b9 --- /dev/null +++ b/java/ql/test/query-tests/SpuriousJavadocParam/options @@ -0,0 +1 @@ +//semmle-extractor-options: --javac-args -source 16 -target 16 diff --git a/java/ql/test/query-tests/SpuriousJavadocParam/test.expected b/java/ql/test/query-tests/SpuriousJavadocParam/test.expected index 42bdb93e60c..0ade1a7d519 100644 --- a/java/ql/test/query-tests/SpuriousJavadocParam/test.expected +++ b/java/ql/test/query-tests/SpuriousJavadocParam/test.expected @@ -12,3 +12,5 @@ | Test.java:112:6:112:12 | @param | @param tag "" does not match any actual type parameter of type "GenericClass". | | Test.java:118:6:118:12 | @param | @param tag "T" does not match any actual type parameter of type "GenericInterface". | | Test.java:119:6:119:12 | @param | @param tag "" does not match any actual type parameter of type "GenericInterface". | +| Test.java:125:6:125:12 | @param | @param tag "k" does not match any actual type parameter or record parameter of record "SomeRecord". | +| Test.java:132:6:132:12 | @param | @param tag "k" does not match any actual type parameter or record parameter of record "GenericRecord". | From 137594cf36f6c3c33ecc4f458d9968c59fdd2e51 Mon Sep 17 00:00:00 2001 From: Tom Hvitved Date: Wed, 3 Apr 2024 14:51:13 +0200 Subject: [PATCH 33/97] Ruby: Add regression test --- .../dataflow/regressions/Regressions.expected | 1 + .../dataflow/regressions/Regressions.ql | 39 +++++++++++++++++++ .../dataflow/regressions/regressions.rb | 3 ++ 3 files changed, 43 insertions(+) create mode 100644 ruby/ql/test/library-tests/dataflow/regressions/Regressions.expected create mode 100644 ruby/ql/test/library-tests/dataflow/regressions/Regressions.ql create mode 100644 ruby/ql/test/library-tests/dataflow/regressions/regressions.rb diff --git a/ruby/ql/test/library-tests/dataflow/regressions/Regressions.expected b/ruby/ql/test/library-tests/dataflow/regressions/Regressions.expected new file mode 100644 index 00000000000..ef846359c3b --- /dev/null +++ b/ruby/ql/test/library-tests/dataflow/regressions/Regressions.expected @@ -0,0 +1 @@ +| regressions.rb:2:1:2:9 | [post] call to reverse | regressions.rb:3:6:3:6 | x | diff --git a/ruby/ql/test/library-tests/dataflow/regressions/Regressions.ql b/ruby/ql/test/library-tests/dataflow/regressions/Regressions.ql new file mode 100644 index 00000000000..2d6c879aa39 --- /dev/null +++ b/ruby/ql/test/library-tests/dataflow/regressions/Regressions.ql @@ -0,0 +1,39 @@ +private import codeql.ruby.dataflow.FlowSummary + +private class ReverseSummary extends SimpleSummarizedCallable { + ReverseSummary() { this = "reverse" } + + override predicate propagatesFlow(string input, string output, boolean preservesValue) { + input = "Argument[self].WithElement[any]" and + output = "ReturnValue" and + preservesValue = true + } +} + +private module Config implements DataFlow::ConfigSig { + predicate isSource(DataFlow::Node source) { + source + .(DataFlow::PostUpdateNode) + .getPreUpdateNode() + .asExpr() + .getExpr() + .(MethodCall) + .getMethodName() = "reverse" + } + + predicate isSink(DataFlow::Node sink) { + exists(MethodCall mc | + mc.getMethodName() = "sink" and + sink.asExpr().getExpr() = mc.getAnArgument() + ) + } +} + +/** + * This predicate should not have a result. We check that the flow summary for + * `reverse` does not get picked up by the `reverseStepThroughInputOutputAlias` + * logic in `DataFlowImplCommon.qll`. + */ +query predicate noReverseStepThroughInputOutputAlias(DataFlow::Node source, DataFlow::Node sink) { + DataFlow::Global::flow(source, sink) +} diff --git a/ruby/ql/test/library-tests/dataflow/regressions/regressions.rb b/ruby/ql/test/library-tests/dataflow/regressions/regressions.rb new file mode 100644 index 00000000000..f3e22585a8d --- /dev/null +++ b/ruby/ql/test/library-tests/dataflow/regressions/regressions.rb @@ -0,0 +1,3 @@ +x = foo +x.reverse.bar +sink(x) \ No newline at end of file From 7871fb8ce66474da9b5e85eb26bf87e7134e8c07 Mon Sep 17 00:00:00 2001 From: Tom Hvitved Date: Wed, 3 Apr 2024 14:54:36 +0200 Subject: [PATCH 34/97] Data flow: Block flow at `expectsContents` nodes in `parameterValueFlow` --- .../dataflow/regressions/Regressions.expected | 1 - .../dataflow/internal/DataFlowImplCommon.qll | 59 ++++++++++--------- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/ruby/ql/test/library-tests/dataflow/regressions/Regressions.expected b/ruby/ql/test/library-tests/dataflow/regressions/Regressions.expected index ef846359c3b..e69de29bb2d 100644 --- a/ruby/ql/test/library-tests/dataflow/regressions/Regressions.expected +++ b/ruby/ql/test/library-tests/dataflow/regressions/Regressions.expected @@ -1 +0,0 @@ -| regressions.rb:2:1:2:9 | [post] call to reverse | regressions.rb:3:6:3:6 | x | diff --git a/shared/dataflow/codeql/dataflow/internal/DataFlowImplCommon.qll b/shared/dataflow/codeql/dataflow/internal/DataFlowImplCommon.qll index e83752fcced..81091303ff4 100644 --- a/shared/dataflow/codeql/dataflow/internal/DataFlowImplCommon.qll +++ b/shared/dataflow/codeql/dataflow/internal/DataFlowImplCommon.qll @@ -863,34 +863,37 @@ module MakeImplCommon Lang> { */ pragma[nomagic] private predicate parameterValueFlowCand(ParamNode p, Node node, boolean read) { - p = node and - read = false - or - // local flow - exists(Node mid | - parameterValueFlowCand(p, mid, read) and - simpleLocalFlowStep(mid, node) and - validParameterAliasStep(mid, node) - ) - or - // read - exists(Node mid | - parameterValueFlowCand(p, mid, false) and - readSet(mid, _, node) and - read = true - ) - or - // flow through: no prior read - exists(ArgNode arg | - parameterValueFlowArgCand(p, arg, false) and - argumentValueFlowsThroughCand(arg, node, read) - ) - or - // flow through: no read inside method - exists(ArgNode arg | - parameterValueFlowArgCand(p, arg, read) and - argumentValueFlowsThroughCand(arg, node, false) - ) + ( + p = node and + read = false + or + // local flow + exists(Node mid | + parameterValueFlowCand(p, mid, read) and + simpleLocalFlowStep(mid, node) and + validParameterAliasStep(mid, node) + ) + or + // read + exists(Node mid | + parameterValueFlowCand(p, mid, false) and + readSet(mid, _, node) and + read = true + ) + or + // flow through: no prior read + exists(ArgNode arg | + parameterValueFlowArgCand(p, arg, false) and + argumentValueFlowsThroughCand(arg, node, read) + ) + or + // flow through: no read inside method + exists(ArgNode arg | + parameterValueFlowArgCand(p, arg, read) and + argumentValueFlowsThroughCand(arg, node, false) + ) + ) and + not expectsContentCached(node, _) } pragma[nomagic] From e42639852c6b26abef54e52ebd5e74eeecbac8f9 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Thu, 4 Apr 2024 14:25:05 +0200 Subject: [PATCH 35/97] C#: Move nuget related `DependencyManager` methods to separate file --- .../DependencyManager.Nuget.cs | 364 ++++++++++++++++++ .../DependencyManager.cs | 351 ----------------- 2 files changed, 364 insertions(+), 351 deletions(-) create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs new file mode 100644 index 00000000000..8e0be8ab141 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs @@ -0,0 +1,364 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Semmle.Util; + +namespace Semmle.Extraction.CSharp.DependencyFetching +{ + public sealed partial class DependencyManager + { + private void RestoreNugetPackages(List allNonBinaryFiles, IEnumerable allProjects, IEnumerable allSolutions, HashSet dllPaths) + { + try + { + using (var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger)) + { + var count = nuget.InstallPackages(); + + if (nuget.PackageCount > 0) + { + CompilationInfos.Add(("packages.config files", nuget.PackageCount.ToString())); + CompilationInfos.Add(("Successfully restored packages.config files", count.ToString())); + } + } + + var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true }); + var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet(); + var excludedPaths = nugetPackageDllPaths + .Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools")) + .ToList(); + + if (nugetPackageDllPaths.Count > 0) + { + logger.LogInfo($"Restored {nugetPackageDllPaths.Count} Nuget DLLs."); + } + if (excludedPaths.Count > 0) + { + logger.LogInfo($"Excluding {excludedPaths.Count} Nuget DLLs."); + } + + foreach (var excludedPath in excludedPaths) + { + logger.LogInfo($"Excluded Nuget DLL: {excludedPath}"); + } + + nugetPackageDllPaths.ExceptWith(excludedPaths); + dllPaths.UnionWith(nugetPackageDllPaths); + } + catch (Exception exc) + { + logger.LogError($"Failed to restore Nuget packages with nuget.exe: {exc.Message}"); + } + + var restoredProjects = RestoreSolutions(allSolutions, out var assets1); + var projects = allProjects.Except(restoredProjects); + RestoreProjects(projects, out var assets2); + + var dependencies = Assets.GetCompilationDependencies(logger, assets1.Union(assets2)); + + var paths = dependencies + .Paths + .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d)) + .ToList(); + dllPaths.UnionWith(paths); + + LogAllUnusedPackages(dependencies); + DownloadMissingPackages(allNonBinaryFiles, dllPaths); + } + + /// + /// Executes `dotnet restore` on all solution files in solutions. + /// As opposed to RestoreProjects this is not run in parallel using PLINQ + /// as `dotnet restore` on a solution already uses multiple threads for restoring + /// the projects (this can be disabled with the `--disable-parallel` flag). + /// Populates assets with the relative paths to the assets files generated by the restore. + /// Returns a list of projects that are up to date with respect to restore. + /// + /// A list of paths to solution files. + private IEnumerable RestoreSolutions(IEnumerable solutions, out IEnumerable assets) + { + var successCount = 0; + var nugetSourceFailures = 0; + var assetFiles = new List(); + var projects = solutions.SelectMany(solution => + { + logger.LogInfo($"Restoring solution {solution}..."); + var res = dotnet.Restore(new(solution, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true)); + if (res.Success) + { + successCount++; + } + if (res.HasNugetPackageSourceError) + { + nugetSourceFailures++; + } + assetFiles.AddRange(res.AssetsFilePaths); + return res.RestoredProjects; + }).ToList(); + assets = assetFiles; + CompilationInfos.Add(("Successfully restored solution files", successCount.ToString())); + CompilationInfos.Add(("Failed solution restore with package source error", nugetSourceFailures.ToString())); + CompilationInfos.Add(("Restored projects through solution files", projects.Count.ToString())); + return projects; + } + + /// + /// Executes `dotnet restore` on all projects in projects. + /// This is done in parallel for performance reasons. + /// Populates assets with the relative paths to the assets files generated by the restore. + /// + /// A list of paths to project files. + private void RestoreProjects(IEnumerable projects, out IEnumerable assets) + { + var successCount = 0; + var nugetSourceFailures = 0; + var assetFiles = new List(); + var sync = new object(); + Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = threads }, project => + { + logger.LogInfo($"Restoring project {project}..."); + var res = dotnet.Restore(new(project, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true)); + lock (sync) + { + if (res.Success) + { + successCount++; + } + if (res.HasNugetPackageSourceError) + { + nugetSourceFailures++; + } + assetFiles.AddRange(res.AssetsFilePaths); + } + }); + assets = assetFiles; + CompilationInfos.Add(("Successfully restored project files", successCount.ToString())); + CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString())); + } + + private void DownloadMissingPackages(List allFiles, ISet dllPaths) + { + var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo); + var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames(); + + var notYetDownloadedPackages = new HashSet(fileContent.AllPackages); + foreach (var alreadyDownloadedPackage in alreadyDownloadedPackages) + { + notYetDownloadedPackages.Remove(new(alreadyDownloadedPackage, PackageReferenceSource.SdkCsProj)); + } + foreach (var alreadyDownloadedLegacyPackage in alreadyDownloadedLegacyPackages) + { + notYetDownloadedPackages.Remove(new(alreadyDownloadedLegacyPackage, PackageReferenceSource.PackagesConfig)); + } + + if (notYetDownloadedPackages.Count == 0) + { + return; + } + + var multipleVersions = notYetDownloadedPackages + .GroupBy(p => p.Name) + .Where(g => g.Count() > 1) + .Select(g => g.Key) + .ToList(); + + foreach (var package in multipleVersions) + { + logger.LogWarning($"Found multiple not yet restored packages with name '{package}'."); + notYetDownloadedPackages.Remove(new(package, PackageReferenceSource.PackagesConfig)); + } + + logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored"); + + var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray(); + string? nugetConfig = null; + if (nugetConfigs.Length > 1) + { + logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}."); + nugetConfig = allFiles + .SelectRootFiles(sourceDir) + .SelectFileNamesByName("nuget.config") + .FirstOrDefault(); + if (nugetConfig == null) + { + logger.LogInfo("Could not find a top-level nuget.config file."); + } + } + else + { + nugetConfig = nugetConfigs.FirstOrDefault(); + } + + if (nugetConfig != null) + { + logger.LogInfo($"Using nuget.config file {nugetConfig}."); + } + + CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString())); + + var successCount = 0; + var sync = new object(); + + Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = threads }, package => + { + var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource); + if (!success) + { + return; + } + + lock (sync) + { + successCount++; + } + }); + + CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString())); + + dllPaths.Add(missingPackageDirectory.DirInfo.FullName); + } + + private void LogAllUnusedPackages(DependencyContainer dependencies) + { + var allPackageDirectories = GetAllPackageDirectories(); + + logger.LogInfo($"Restored {allPackageDirectories.Count} packages"); + logger.LogInfo($"Found {dependencies.Packages.Count} packages in project.assets.json files"); + + allPackageDirectories + .Where(package => !dependencies.Packages.Contains(package)) + .Order() + .ForEach(package => logger.LogInfo($"Unused package: {package}")); + } + + + private ICollection GetAllPackageDirectories() + { + return new DirectoryInfo(packageDirectory.DirInfo.FullName) + .EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false }) + .Select(d => d.Name) + .ToList(); + } + + private static bool IsPathInSubfolder(string path, string rootFolder, string subFolder) + { + return path.IndexOf( + $"{Path.DirectorySeparatorChar}{subFolder}{Path.DirectorySeparatorChar}", + rootFolder.Length, + StringComparison.InvariantCultureIgnoreCase) >= 0; + } + + private IEnumerable GetRestoredLegacyPackageNames() + { + var oldPackageDirectories = GetRestoredPackageDirectoryNames(legacyPackageDirectory.DirInfo); + foreach (var oldPackageDirectory in oldPackageDirectories) + { + // nuget install restores packages to 'packagename.version' folders (dotnet restore to 'packagename/version' folders) + // typical folder names look like: + // newtonsoft.json.13.0.3 + // there are more complex ones too, such as: + // runtime.tizen.4.0.0-armel.Microsoft.NETCore.DotNetHostResolver.2.0.0-preview2-25407-01 + + var match = LegacyNugetPackage().Match(oldPackageDirectory); + if (!match.Success) + { + logger.LogWarning($"Package directory '{oldPackageDirectory}' doesn't match the expected pattern."); + continue; + } + + yield return match.Groups[1].Value.ToLowerInvariant(); + } + } + + private static IEnumerable GetRestoredPackageDirectoryNames(DirectoryInfo root) + { + return Directory.GetDirectories(root.FullName) + .Select(d => Path.GetFileName(d).ToLowerInvariant()); + } + + [GeneratedRegex(@".*", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)] + private static partial Regex TargetFramework(); + + private bool TryRestorePackageManually(string package, string? nugetConfig, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj) + { + logger.LogInfo($"Restoring package {package}..."); + using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir")); + var success = dotnet.New(tempDir.DirInfo.FullName); + if (!success) + { + return false; + } + + if (packageReferenceSource == PackageReferenceSource.PackagesConfig) + { + TryChangeTargetFrameworkMoniker(tempDir.DirInfo); + } + + success = dotnet.AddPackage(tempDir.DirInfo.FullName, package); + if (!success) + { + return false; + } + + var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig)); + if (!res.Success) + { + if (res.HasNugetPackageSourceError && nugetConfig is not null) + { + // Restore could not be completed because the listed source is unavailable. Try without the nuget.config: + res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true)); + } + + // TODO: the restore might fail, we could retry with + // - a prerelease (*-* instead of *) version of the package, + // - a different target framework moniker. + + if (!res.Success) + { + logger.LogInfo($"Failed to restore nuget package {package}"); + return false; + } + } + + return true; + } + + private void TryChangeTargetFrameworkMoniker(DirectoryInfo tempDir) + { + try + { + logger.LogInfo($"Changing the target framework moniker in {tempDir.FullName}..."); + + var csprojs = tempDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive }); + if (csprojs.Length != 1) + { + logger.LogError($"Could not find the .csproj file in {tempDir.FullName}, count = {csprojs.Length}"); + return; + } + + var csproj = csprojs[0]; + var content = File.ReadAllText(csproj.FullName); + var matches = TargetFramework().Matches(content); + if (matches.Count == 0) + { + logger.LogError($"Could not find target framework in {csproj.FullName}"); + } + else + { + content = TargetFramework().Replace(content, $"{FrameworkPackageNames.LatestNetFrameworkMoniker}", 1); + File.WriteAllText(csproj.FullName, content); + } + } + catch (Exception exc) + { + logger.LogError($"Failed to update target framework in {tempDir.FullName}: {exc}"); + } + } + + [GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)] + private static partial Regex LegacyNugetPackage(); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index 713d52d8e3f..04716a64db4 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -230,73 +230,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return frameworkLocations; } - private void RestoreNugetPackages(List allNonBinaryFiles, IEnumerable allProjects, IEnumerable allSolutions, HashSet dllPaths) - { - try - { - using (var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger)) - { - var count = nuget.InstallPackages(); - - if (nuget.PackageCount > 0) - { - CompilationInfos.Add(("packages.config files", nuget.PackageCount.ToString())); - CompilationInfos.Add(("Successfully restored packages.config files", count.ToString())); - } - } - - var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true }); - var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet(); - var excludedPaths = nugetPackageDllPaths - .Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools")) - .ToList(); - - if (nugetPackageDllPaths.Count > 0) - { - logger.LogInfo($"Restored {nugetPackageDllPaths.Count} Nuget DLLs."); - } - if (excludedPaths.Count > 0) - { - logger.LogInfo($"Excluding {excludedPaths.Count} Nuget DLLs."); - } - - foreach (var excludedPath in excludedPaths) - { - logger.LogInfo($"Excluded Nuget DLL: {excludedPath}"); - } - - nugetPackageDllPaths.ExceptWith(excludedPaths); - dllPaths.UnionWith(nugetPackageDllPaths); - } - catch (Exception exc) - { - logger.LogError($"Failed to restore Nuget packages with nuget.exe: {exc.Message}"); - } - - var restoredProjects = RestoreSolutions(allSolutions, out var assets1); - var projects = allProjects.Except(restoredProjects); - RestoreProjects(projects, out var assets2); - - var dependencies = Assets.GetCompilationDependencies(logger, assets1.Union(assets2)); - - var paths = dependencies - .Paths - .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d)) - .ToList(); - dllPaths.UnionWith(paths); - - LogAllUnusedPackages(dependencies); - DownloadMissingPackages(allNonBinaryFiles, dllPaths); - } - - private static bool IsPathInSubfolder(string path, string rootFolder, string subFolder) - { - return path.IndexOf( - $"{Path.DirectorySeparatorChar}{subFolder}{Path.DirectorySeparatorChar}", - rootFolder.Length, - StringComparison.InvariantCultureIgnoreCase) >= 0; - } - private void RemoveNugetAnalyzerReferences() { var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant(); @@ -483,27 +416,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching .FullName; } - private ICollection GetAllPackageDirectories() - { - return new DirectoryInfo(packageDirectory.DirInfo.FullName) - .EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false }) - .Select(d => d.Name) - .ToList(); - } - - private void LogAllUnusedPackages(DependencyContainer dependencies) - { - var allPackageDirectories = GetAllPackageDirectories(); - - logger.LogInfo($"Restored {allPackageDirectories.Count} packages"); - logger.LogInfo($"Found {dependencies.Packages.Count} packages in project.assets.json files"); - - allPackageDirectories - .Where(package => !dependencies.Packages.Contains(package)) - .Order() - .ForEach(package => logger.LogInfo($"Unused package: {package}")); - } - private void GenerateSourceFileFromImplicitUsings() { var usings = new HashSet(); @@ -807,269 +719,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - /// - /// Executes `dotnet restore` on all solution files in solutions. - /// As opposed to RestoreProjects this is not run in parallel using PLINQ - /// as `dotnet restore` on a solution already uses multiple threads for restoring - /// the projects (this can be disabled with the `--disable-parallel` flag). - /// Populates assets with the relative paths to the assets files generated by the restore. - /// Returns a list of projects that are up to date with respect to restore. - /// - /// A list of paths to solution files. - private IEnumerable RestoreSolutions(IEnumerable solutions, out IEnumerable assets) - { - var successCount = 0; - var nugetSourceFailures = 0; - var assetFiles = new List(); - var projects = solutions.SelectMany(solution => - { - logger.LogInfo($"Restoring solution {solution}..."); - var res = dotnet.Restore(new(solution, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true)); - if (res.Success) - { - successCount++; - } - if (res.HasNugetPackageSourceError) - { - nugetSourceFailures++; - } - assetFiles.AddRange(res.AssetsFilePaths); - return res.RestoredProjects; - }).ToList(); - assets = assetFiles; - CompilationInfos.Add(("Successfully restored solution files", successCount.ToString())); - CompilationInfos.Add(("Failed solution restore with package source error", nugetSourceFailures.ToString())); - CompilationInfos.Add(("Restored projects through solution files", projects.Count.ToString())); - return projects; - } - - /// - /// Executes `dotnet restore` on all projects in projects. - /// This is done in parallel for performance reasons. - /// Populates assets with the relative paths to the assets files generated by the restore. - /// - /// A list of paths to project files. - private void RestoreProjects(IEnumerable projects, out IEnumerable assets) - { - var successCount = 0; - var nugetSourceFailures = 0; - var assetFiles = new List(); - var sync = new object(); - Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = threads }, project => - { - logger.LogInfo($"Restoring project {project}..."); - var res = dotnet.Restore(new(project, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true)); - lock (sync) - { - if (res.Success) - { - successCount++; - } - if (res.HasNugetPackageSourceError) - { - nugetSourceFailures++; - } - assetFiles.AddRange(res.AssetsFilePaths); - } - }); - assets = assetFiles; - CompilationInfos.Add(("Successfully restored project files", successCount.ToString())); - CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString())); - } - - [GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)] - private static partial Regex LegacyNugetPackage(); - - - private static IEnumerable GetRestoredPackageDirectoryNames(DirectoryInfo root) - { - return Directory.GetDirectories(root.FullName) - .Select(d => Path.GetFileName(d).ToLowerInvariant()); - } - - private IEnumerable GetRestoredLegacyPackageNames() - { - var oldPackageDirectories = GetRestoredPackageDirectoryNames(legacyPackageDirectory.DirInfo); - foreach (var oldPackageDirectory in oldPackageDirectories) - { - // nuget install restores packages to 'packagename.version' folders (dotnet restore to 'packagename/version' folders) - // typical folder names look like: - // newtonsoft.json.13.0.3 - // there are more complex ones too, such as: - // runtime.tizen.4.0.0-armel.Microsoft.NETCore.DotNetHostResolver.2.0.0-preview2-25407-01 - - var match = LegacyNugetPackage().Match(oldPackageDirectory); - if (!match.Success) - { - logger.LogWarning($"Package directory '{oldPackageDirectory}' doesn't match the expected pattern."); - continue; - } - - yield return match.Groups[1].Value.ToLowerInvariant(); - } - } - - private void DownloadMissingPackages(List allFiles, ISet dllPaths) - { - var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo); - var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames(); - - var notYetDownloadedPackages = new HashSet(fileContent.AllPackages); - foreach (var alreadyDownloadedPackage in alreadyDownloadedPackages) - { - notYetDownloadedPackages.Remove(new(alreadyDownloadedPackage, PackageReferenceSource.SdkCsProj)); - } - foreach (var alreadyDownloadedLegacyPackage in alreadyDownloadedLegacyPackages) - { - notYetDownloadedPackages.Remove(new(alreadyDownloadedLegacyPackage, PackageReferenceSource.PackagesConfig)); - } - - if (notYetDownloadedPackages.Count == 0) - { - return; - } - - var multipleVersions = notYetDownloadedPackages - .GroupBy(p => p.Name) - .Where(g => g.Count() > 1) - .Select(g => g.Key) - .ToList(); - - foreach (var package in multipleVersions) - { - logger.LogWarning($"Found multiple not yet restored packages with name '{package}'."); - notYetDownloadedPackages.Remove(new(package, PackageReferenceSource.PackagesConfig)); - } - - logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored"); - - var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray(); - string? nugetConfig = null; - if (nugetConfigs.Length > 1) - { - logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}."); - nugetConfig = allFiles - .SelectRootFiles(sourceDir) - .SelectFileNamesByName("nuget.config") - .FirstOrDefault(); - if (nugetConfig == null) - { - logger.LogInfo("Could not find a top-level nuget.config file."); - } - } - else - { - nugetConfig = nugetConfigs.FirstOrDefault(); - } - - if (nugetConfig != null) - { - logger.LogInfo($"Using nuget.config file {nugetConfig}."); - } - - CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString())); - - var successCount = 0; - var sync = new object(); - - Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = threads }, package => - { - var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource); - if (!success) - { - return; - } - - lock (sync) - { - successCount++; - } - }); - - CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString())); - - dllPaths.Add(missingPackageDirectory.DirInfo.FullName); - } - - [GeneratedRegex(@".*", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)] - private static partial Regex TargetFramework(); - - private bool TryRestorePackageManually(string package, string? nugetConfig, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj) - { - logger.LogInfo($"Restoring package {package}..."); - using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir")); - var success = dotnet.New(tempDir.DirInfo.FullName); - if (!success) - { - return false; - } - - if (packageReferenceSource == PackageReferenceSource.PackagesConfig) - { - TryChangeTargetFrameworkMoniker(tempDir.DirInfo); - } - - success = dotnet.AddPackage(tempDir.DirInfo.FullName, package); - if (!success) - { - return false; - } - - var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig)); - if (!res.Success) - { - if (res.HasNugetPackageSourceError && nugetConfig is not null) - { - // Restore could not be completed because the listed source is unavailable. Try without the nuget.config: - res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true)); - } - - // TODO: the restore might fail, we could retry with - // - a prerelease (*-* instead of *) version of the package, - // - a different target framework moniker. - - if (!res.Success) - { - logger.LogInfo($"Failed to restore nuget package {package}"); - return false; - } - } - - return true; - } - - private void TryChangeTargetFrameworkMoniker(DirectoryInfo tempDir) - { - try - { - logger.LogInfo($"Changing the target framework moniker in {tempDir.FullName}..."); - - var csprojs = tempDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive }); - if (csprojs.Length != 1) - { - logger.LogError($"Could not find the .csproj file in {tempDir.FullName}, count = {csprojs.Length}"); - return; - } - - var csproj = csprojs[0]; - var content = File.ReadAllText(csproj.FullName); - var matches = TargetFramework().Matches(content); - if (matches.Count == 0) - { - logger.LogError($"Could not find target framework in {csproj.FullName}"); - } - else - { - content = TargetFramework().Replace(content, $"{FrameworkPackageNames.LatestNetFrameworkMoniker}", 1); - File.WriteAllText(csproj.FullName, content); - } - } - catch (Exception exc) - { - logger.LogError($"Failed to update target framework in {tempDir.FullName}: {exc}"); - } - } - public void Dispose(TemporaryDirectory? dir, string name) { try From 2336e1462749386aa3db178df112d81c2f569e09 Mon Sep 17 00:00:00 2001 From: Dave Bartolomeo Date: Thu, 4 Apr 2024 10:31:05 -0400 Subject: [PATCH 36/97] Remove expectation of spurious diagnostic --- java/ql/test/query-tests/SpuriousJavadocParam/Test.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/java/ql/test/query-tests/SpuriousJavadocParam/Test.java b/java/ql/test/query-tests/SpuriousJavadocParam/Test.java index de108008ee8..434353f4c67 100644 --- a/java/ql/test/query-tests/SpuriousJavadocParam/Test.java +++ b/java/ql/test/query-tests/SpuriousJavadocParam/Test.java @@ -132,6 +132,4 @@ public class Test { * @param k does not */ static record GenericRecord(int i, int j) {} - - // Diagnostic Matches: Incomplete inheritance relation for type java.lang.Object and supertype none } From 7bec41096c182364830c260ce49489d593ca9550 Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 3 Apr 2024 14:18:30 +0000 Subject: [PATCH 37/97] Python: Rename `tsg-build` target to `tsp-build` The latter makes more sense, as it's actually building `tree-sitter-python`. --- python/extractor/tsg-python/tsp/BUILD.bazel | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/extractor/tsg-python/tsp/BUILD.bazel b/python/extractor/tsg-python/tsp/BUILD.bazel index 71319e894f6..e3389fce1cc 100644 --- a/python/extractor/tsg-python/tsp/BUILD.bazel +++ b/python/extractor/tsg-python/tsp/BUILD.bazel @@ -7,7 +7,7 @@ package(default_visibility = ["//visibility:public"]) # This will run the build script from the root of the workspace, and # collect the outputs. cargo_build_script( - name = "tsg-build", + name = "tsp-build", srcs = ["bindings/rust/build.rs"], data = glob([ "src/**", @@ -32,7 +32,7 @@ rust_library( proc_macro_deps = all_crate_deps( proc_macro = True, ), - deps = [":tsg-build"] + all_crate_deps( + deps = [":tsp-build"] + all_crate_deps( normal = True, ), ) From 752d28c1b9fee6047180b7233e753473028b557a Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 3 Apr 2024 14:20:54 +0000 Subject: [PATCH 38/97] Python: Update repinning instructions This aligns us better with the corresponding instructions for the Ruby extractor. --- python/extractor/tsg-python/Cargo.toml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/python/extractor/tsg-python/Cargo.toml b/python/extractor/tsg-python/Cargo.toml index feecd254159..ac4d4093189 100644 --- a/python/extractor/tsg-python/Cargo.toml +++ b/python/extractor/tsg-python/Cargo.toml @@ -7,9 +7,14 @@ authors = ["Taus Brock-Nannestad "] edition = "2018" # When changing/updating these, the `Cargo.Bazel.lock` file has to be regenerated. -# Check out the documentation at https://bazelbuild.github.io/rules_rust/crate_universe.html#repinning--updating-dependencies -# for how to do so. The bazel repository for the tsg-python project is called `py_deps`, -# and instead of calling `bazel sync`, `./build --bazel sync` should be used instead, to always use the correct bazel version. +# Run `CARGO_BAZEL_REPIN=true CARGO_BAZEL_REPIN_ONLY=py_deps ./build --bazel sync --only=py_deps` +# in the `semmle-code` repository to do so. +# For more information, check out the documentation at +# https://bazelbuild.github.io/rules_rust/crate_universe.html#repinning--updating-dependencies +# In the future, the hope is to move this handling of the dependencies entirely into the `codeql` repository, +# but that depends on `rules_rust` being fully compatible with bzlmod, which they aren't yet +# (c.f. https://github.com/bazelbuild/rules_rust/issues/2452). +# Warning: The process takes >5min on my M1 mac, so do wait for a while. [dependencies] anyhow = "1.0" regex = "1" From 599f573a4abb0b20411aa22298dd3ec0c5a1eaf1 Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 3 Apr 2024 14:25:03 +0000 Subject: [PATCH 39/97] Python: Preserve comments and docstrings in extractor --- python/extractor/make_zips.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/python/extractor/make_zips.py b/python/extractor/make_zips.py index b91b1bf458d..faf595acc37 100755 --- a/python/extractor/make_zips.py +++ b/python/extractor/make_zips.py @@ -8,7 +8,6 @@ import optparse import compileall from python_tracer import getzipfilename -from unparse import strip_comments_and_docstrings # TO DO -- Add options to set destination directory and source directory @@ -84,9 +83,7 @@ def write_source(zipped, root, name, extensions=[".py"]): if ext not in extensions: continue path = os.path.join(dirpath, name) - temp = strip_comments_and_docstrings(path) - zipped.write(temp, os.path.relpath(path, root)) - os.remove(temp) + zipped.write(path, os.path.relpath(path, root)) def main(): parser = optparse.OptionParser(usage = "usage: %prog [install-dir]") From ef9f99b3be728d910539c7750e77a11cfda64425 Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 3 Apr 2024 14:30:29 +0000 Subject: [PATCH 40/97] Python: Remove `unparse.py` --- python/extractor/BUILD.bazel | 1 - python/extractor/licenses.md | 1 - python/extractor/unparse.py | 709 ----------------------------------- 3 files changed, 711 deletions(-) delete mode 100644 python/extractor/unparse.py diff --git a/python/extractor/BUILD.bazel b/python/extractor/BUILD.bazel index 3b5a5b3a617..697bf8d49a4 100644 --- a/python/extractor/BUILD.bazel +++ b/python/extractor/BUILD.bazel @@ -5,7 +5,6 @@ py_binary( srcs = [ "make_zips.py", "python_tracer.py", - "unparse.py", ], data = [ "LICENSE-PSF.md", diff --git a/python/extractor/licenses.md b/python/extractor/licenses.md index f3a5c6cc458..d585a84626d 100644 --- a/python/extractor/licenses.md +++ b/python/extractor/licenses.md @@ -6,7 +6,6 @@ | `tsg-python/tree-sitter-python` | Y | MIT | Used in `tsg-python` to parse Python files | | `tsg-python` | Y | MIT / Apache | This is our own creation, so are free to choose what license it is covered by. | | `tree-sitter-graph` | N | MIT / Apache | Used in `tsg-python` to execute files written in the `tree-sitter-graph` language. | -| `unparse.py` | Y | PSF | Copied and adapted from `Tools/unparse.py` from the `cpython` source code, with attribution. | | `imp.py` | Y | PSF | Copied and adapted from `Lib/imp.py` from the `cpython` source code, with attribution. | | `semmle/data/*.trap` | Y | PSF | These files were derived from the C source code of the `cpython` project, and are used in our modelling of built-in objects. No attribution, currently. | | `semmle/thrift/parse.py` | Y | Apache | Includes a grammar based on https://github.com/apache/thrift/blob/master/doc/specs/idl.md, with comment stating this attribution. | diff --git a/python/extractor/unparse.py b/python/extractor/unparse.py deleted file mode 100644 index b12f1501592..00000000000 --- a/python/extractor/unparse.py +++ /dev/null @@ -1,709 +0,0 @@ -#Copied Tools.unparse.py with modifications. Copyright PSF. - -"Usage: unparse.py " -import sys -import ast -import tokenize -import io -import os -import shutil - -# Large float and imaginary literals get turned into infinities in the AST. -# We unparse those infinities to INFSTR. -INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) - -def interleave(inter, f, seq): - """Call f on each item in seq, calling inter() in between. - """ - seq = iter(seq) - try: - f(next(seq)) - except StopIteration: - pass - else: - for x in seq: - inter() - f(x) - -class Unparser: - """Methods in this class recursively traverse an AST and - output source code for the abstract syntax; original formatting - is disregarded. """ - - def __init__(self, tree, file = sys.stdout): - """Unparser(tree, file=sys.stdout) -> None. - Print the source for tree to file.""" - self.f = file - self._indent = 0 - self.dispatch(tree) - print("", file=self.f) - self.f.flush() - - def fill(self, text = ""): - "Indent a piece of text, according to the current indentation level" - self.f.write("\n"+" "*self._indent + text) - - def write(self, text): - "Append a piece of text to the current line." - self.f.write(text) - - def enter(self): - "Print ':', and increase the indentation." - self.write(":") - self._indent += 1 - - def leave(self): - "Decrease the indentation level." - self._indent -= 1 - - def dispatch(self, tree): - "Dispatcher function, dispatching tree type T to method _T." - if isinstance(tree, list): - for t in tree: - self.dispatch(t) - return - meth = getattr(self, "_"+tree.__class__.__name__) - meth(tree) - - def remove_docstring(self, t): - if hasattr(t, "docstring"): - return - if not t.body: - return - if not isinstance(t.body[0], ast.Expr): - return - if not isinstance(t.body[0].value, ast.Str): - return - t.body = t.body[1:] - - def add_pass(self, t): - if t.body: - #No pass needed - return - t.body = [ast.Pass()] - - ############### Unparsing methods ###################### - # There should be one method per concrete grammar type # - # Constructors should be grouped by sum type. Ideally, # - # this would follow the order in the grammar, but # - # currently doesn't. # - ######################################################## - - def _Module(self, tree): - self.remove_docstring(tree) - self.add_pass(tree) - for stmt in tree.body: - self.dispatch(stmt) - - # stmt - def _Expr(self, tree): - self.fill() - self.dispatch(tree.value) - - def _Import(self, t): - self.fill("import ") - interleave(lambda: self.write(", "), self.dispatch, t.names) - - def _ImportFrom(self, t): - self.fill("from ") - self.write("." * t.level) - if t.module: - self.write(t.module) - self.write(" import ") - interleave(lambda: self.write(", "), self.dispatch, t.names) - - def _Assign(self, t): - self.fill() - for target in t.targets: - self.dispatch(target) - self.write(" = ") - self.dispatch(t.value) - - def _AugAssign(self, t): - self.fill() - self.dispatch(t.target) - self.write(" "+self.binop[t.op.__class__.__name__]+"= ") - self.dispatch(t.value) - - def _AnnAssign(self, t): - self.fill() - if not t.simple and isinstance(t.target, ast.Name): - self.write('(') - self.dispatch(t.target) - if not t.simple and isinstance(t.target, ast.Name): - self.write(')') - self.write(": ") - self.dispatch(t.annotation) - if t.value: - self.write(" = ") - self.dispatch(t.value) - - def _Return(self, t): - self.fill("return") - if t.value: - self.write(" ") - self.dispatch(t.value) - - def _Pass(self, t): - self.fill("pass") - - def _Break(self, t): - self.fill("break") - - def _Continue(self, t): - self.fill("continue") - - def _Delete(self, t): - self.fill("del ") - interleave(lambda: self.write(", "), self.dispatch, t.targets) - - def _Assert(self, t): - self.fill("assert ") - self.dispatch(t.test) - if t.msg: - self.write(", ") - self.dispatch(t.msg) - - def _Global(self, t): - self.fill("global ") - interleave(lambda: self.write(", "), self.write, t.names) - - def _Nonlocal(self, t): - self.fill("nonlocal ") - interleave(lambda: self.write(", "), self.write, t.names) - - def _Await(self, t): - self.write("(") - self.write("await") - if t.value: - self.write(" ") - self.dispatch(t.value) - self.write(")") - - def _Yield(self, t): - self.write("(") - self.write("yield") - if t.value: - self.write(" ") - self.dispatch(t.value) - self.write(")") - - def _YieldFrom(self, t): - self.write("(") - self.write("yield from") - if t.value: - self.write(" ") - self.dispatch(t.value) - self.write(")") - - def _Raise(self, t): - self.fill("raise") - if not t.exc: - assert not t.cause - return - self.write(" ") - self.dispatch(t.exc) - if t.cause: - self.write(" from ") - self.dispatch(t.cause) - - def _Try(self, t): - self.fill("try") - self.enter() - self.dispatch(t.body) - self.leave() - for ex in t.handlers: - self.dispatch(ex) - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - if t.finalbody: - self.fill("finally") - self.enter() - self.dispatch(t.finalbody) - self.leave() - - def _ExceptHandler(self, t): - self.fill("except") - if t.type: - self.write(" ") - self.dispatch(t.type) - if t.name: - self.write(" as ") - self.write(t.name) - self.enter() - self.dispatch(t.body) - self.leave() - - def _ClassDef(self, t): - self.write("\n") - for deco in t.decorator_list: - self.fill("@") - self.dispatch(deco) - self.fill("class "+t.name) - self.write("(") - comma = False - for e in t.bases: - if comma: self.write(", ") - else: comma = True - self.dispatch(e) - for e in t.keywords: - if comma: self.write(", ") - else: comma = True - self.dispatch(e) - self.write(")") - - self.enter() - self.remove_docstring(t) - self.add_pass(t) - self.dispatch(t.body) - self.leave() - - def _FunctionDef(self, t): - self.__FunctionDef_helper(t, "def") - - def _AsyncFunctionDef(self, t): - self.__FunctionDef_helper(t, "async def") - - def __FunctionDef_helper(self, t, fill_suffix): - self.write("\n") - for deco in t.decorator_list: - self.fill("@") - self.dispatch(deco) - def_str = fill_suffix+" "+t.name + "(" - self.fill(def_str) - self.dispatch(t.args) - self.write(")") - if t.returns: - self.write(" -> ") - self.dispatch(t.returns) - self.enter() - self.remove_docstring(t) - self.add_pass(t) - self.dispatch(t.body) - self.leave() - - def _For(self, t): - self.__For_helper("for ", t) - - def _AsyncFor(self, t): - self.__For_helper("async for ", t) - - def __For_helper(self, fill, t): - self.fill(fill) - self.dispatch(t.target) - self.write(" in ") - self.dispatch(t.iter) - self.enter() - self.dispatch(t.body) - self.leave() - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - - def _If(self, t): - self.fill("if ") - self.dispatch(t.test) - self.enter() - self.dispatch(t.body) - self.leave() - # collapse nested ifs into equivalent elifs. - while (t.orelse and len(t.orelse) == 1 and - isinstance(t.orelse[0], ast.If)): - t = t.orelse[0] - self.fill("elif ") - self.dispatch(t.test) - self.enter() - self.dispatch(t.body) - self.leave() - # final else - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - - def _While(self, t): - self.fill("while ") - self.dispatch(t.test) - self.enter() - self.dispatch(t.body) - self.leave() - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - - def _With(self, t): - self.fill("with ") - interleave(lambda: self.write(", "), self.dispatch, t.items) - self.enter() - self.dispatch(t.body) - self.leave() - - def _AsyncWith(self, t): - self.fill("async with ") - interleave(lambda: self.write(", "), self.dispatch, t.items) - self.enter() - self.dispatch(t.body) - self.leave() - - # expr - def _Bytes(self, t): - self.write(repr(t.s)) - - def _Str(self, tree): - s = repr(tree.s).encode("ascii", errors="backslashreplace").decode("ascii") - self.write(s) - - def _JoinedStr(self, t): - self.write("f") - string = io.StringIO() - self._fstring_JoinedStr(t, string.write) - self.write(repr(string.getvalue())) - - def _FormattedValue(self, t): - self.write("f") - string = io.StringIO() - self._fstring_FormattedValue(t, string.write) - self.write(repr(string.getvalue())) - - def _fstring_JoinedStr(self, t, write): - for value in t.values: - meth = getattr(self, "_fstring_" + type(value).__name__) - meth(value, write) - - def _fstring_Str(self, t, write): - value = t.s.replace("{", "{{").replace("}", "}}") - write(value) - - def _fstring_Constant(self, t, write): - assert isinstance(t.value, str) - value = t.value.replace("{", "{{").replace("}", "}}") - write(value) - - def _fstring_FormattedValue(self, t, write): - write("{") - expr = io.StringIO() - Unparser(t.value, expr) - expr = expr.getvalue().rstrip("\n") - if expr.startswith("{"): - write(" ") # Separate pair of opening brackets as "{ {" - write(expr) - if t.conversion != -1: - conversion = chr(t.conversion) - assert conversion in "sra" - write("!%s" % conversion) - if t.format_spec: - write(":") - meth = getattr(self, "_fstring_" + type(t.format_spec).__name__) - meth(t.format_spec, write) - write("}") - - def _Name(self, t): - self.write(t.id) - - def _write_constant(self, value): - if isinstance(value, (float, complex)): - self.write(repr(value).replace("inf", INFSTR)) - else: - self.write(repr(value)) - - def _Constant(self, t): - value = t.value - if isinstance(value, tuple): - self.write("(") - if len(value) == 1: - self._write_constant(value[0]) - self.write(",") - else: - interleave(lambda: self.write(", "), self._write_constant, value) - self.write(")") - else: - self._write_constant(t.value) - - def _NameConstant(self, t): - self.write(repr(t.value)) - - def _Num(self, t): - # Substitute overflowing decimal literal for AST infinities. - self.write(repr(t.n).replace("inf", INFSTR)) - - def _List(self, t): - self.write("[") - interleave(lambda: self.write(", "), self.dispatch, t.elts) - self.write("]") - - def _ListComp(self, t): - self.write("[") - self.dispatch(t.elt) - for gen in t.generators: - self.dispatch(gen) - self.write("]") - - def _GeneratorExp(self, t): - self.write("(") - self.dispatch(t.elt) - for gen in t.generators: - self.dispatch(gen) - self.write(")") - - def _SetComp(self, t): - self.write("{") - self.dispatch(t.elt) - for gen in t.generators: - self.dispatch(gen) - self.write("}") - - def _DictComp(self, t): - self.write("{") - self.dispatch(t.key) - self.write(": ") - self.dispatch(t.value) - for gen in t.generators: - self.dispatch(gen) - self.write("}") - - def _comprehension(self, t): - if hasattr(t, "is_async") and t.is_async: - self.write(" async for ") - else: - self.write(" for ") - self.dispatch(t.target) - self.write(" in ") - self.dispatch(t.iter) - for if_clause in t.ifs: - self.write(" if ") - self.dispatch(if_clause) - - def _IfExp(self, t): - self.write("(") - self.dispatch(t.body) - self.write(" if ") - self.dispatch(t.test) - self.write(" else ") - self.dispatch(t.orelse) - self.write(")") - - def _Set(self, t): - assert(t.elts) # should be at least one element - self.write("{") - interleave(lambda: self.write(", "), self.dispatch, t.elts) - self.write("}") - - def _Dict(self, t): - self.write("{") - def write_key_value_pair(k, v): - self.dispatch(k) - self.write(": ") - self.dispatch(v) - - def write_item(item): - k, v = item - if k is None: - # for dictionary unpacking operator in dicts {**{'y': 2}} - # see PEP 448 for details - self.write("**") - self.dispatch(v) - else: - write_key_value_pair(k, v) - interleave(lambda: self.write(", "), write_item, zip(t.keys, t.values)) - self.write("}") - - def _Tuple(self, t): - self.write("(") - if len(t.elts) == 1: - elt = t.elts[0] - self.dispatch(elt) - self.write(",") - else: - interleave(lambda: self.write(", "), self.dispatch, t.elts) - self.write(")") - - unop = {"Invert":"~", "Not": "not", "UAdd":"+", "USub":"-"} - def _UnaryOp(self, t): - self.write("(") - self.write(self.unop[t.op.__class__.__name__]) - self.write(" ") - self.dispatch(t.operand) - self.write(")") - - binop = { "Add":"+", "Sub":"-", "Mult":"*", "MatMult":"@", "Div":"/", "Mod":"%", - "LShift":"<<", "RShift":">>", "BitOr":"|", "BitXor":"^", "BitAnd":"&", - "FloorDiv":"//", "Pow": "**"} - def _BinOp(self, t): - self.write("(") - self.dispatch(t.left) - self.write(" " + self.binop[t.op.__class__.__name__] + " ") - self.dispatch(t.right) - self.write(")") - - cmpops = {"Eq":"==", "NotEq":"!=", "Lt":"<", "LtE":"<=", "Gt":">", "GtE":">=", - "Is":"is", "IsNot":"is not", "In":"in", "NotIn":"not in"} - def _Compare(self, t): - self.write("(") - self.dispatch(t.left) - for o, e in zip(t.ops, t.comparators): - self.write(" " + self.cmpops[o.__class__.__name__] + " ") - self.dispatch(e) - self.write(")") - - boolops = {ast.And: 'and', ast.Or: 'or'} - def _BoolOp(self, t): - self.write("(") - s = " %s " % self.boolops[t.op.__class__] - interleave(lambda: self.write(s), self.dispatch, t.values) - self.write(")") - - def _Attribute(self,t): - self.dispatch(t.value) - # Special case: 3.__abs__() is a syntax error, so if t.value - # is an integer literal then we need to either parenthesize - # it or add an extra space to get 3 .__abs__(). - if isinstance(t.value, ast.Num) and isinstance(t.value.n, int): - self.write(" ") - self.write(".") - self.write(t.attr) - - def _Call(self, t): - self.dispatch(t.func) - self.write("(") - comma = False - for e in t.args: - if comma: self.write(", ") - else: comma = True - self.dispatch(e) - for e in t.keywords: - if comma: self.write(", ") - else: comma = True - self.dispatch(e) - self.write(")") - - def _Subscript(self, t): - self.dispatch(t.value) - self.write("[") - self.dispatch(t.slice) - self.write("]") - - def _Starred(self, t): - self.write("*") - self.dispatch(t.value) - - # slice - def _Ellipsis(self, t): - self.write("...") - - def _Index(self, t): - self.dispatch(t.value) - - def _Slice(self, t): - if t.lower: - self.dispatch(t.lower) - self.write(":") - if t.upper: - self.dispatch(t.upper) - if t.step: - self.write(":") - self.dispatch(t.step) - - def _ExtSlice(self, t): - interleave(lambda: self.write(', '), self.dispatch, t.dims) - - # argument - def _arg(self, t): - self.write(t.arg) - if t.annotation: - self.write(": ") - self.dispatch(t.annotation) - - # others - def _arguments(self, t): - first = True - # normal arguments - defaults = [None] * (len(t.args) - len(t.defaults)) + t.defaults - for a, d in zip(t.args, defaults): - if first:first = False - else: self.write(", ") - self.dispatch(a) - if d: - self.write("=") - self.dispatch(d) - - # varargs, or bare '*' if no varargs but keyword-only arguments present - if t.vararg or t.kwonlyargs: - if first:first = False - else: self.write(", ") - self.write("*") - if t.vararg: - self.write(t.vararg.arg) - if t.vararg.annotation: - self.write(": ") - self.dispatch(t.vararg.annotation) - - # keyword-only arguments - if t.kwonlyargs: - for a, d in zip(t.kwonlyargs, t.kw_defaults): - if first:first = False - else: self.write(", ") - self.dispatch(a), - if d: - self.write("=") - self.dispatch(d) - - # kwargs - if t.kwarg: - if first:first = False - else: self.write(", ") - self.write("**"+t.kwarg.arg) - if t.kwarg.annotation: - self.write(": ") - self.dispatch(t.kwarg.annotation) - - def _keyword(self, t): - if t.arg is None: - self.write("**") - else: - self.write(t.arg) - self.write("=") - self.dispatch(t.value) - - def _Lambda(self, t): - self.write("(") - self.write("lambda ") - self.dispatch(t.args) - self.write(": ") - self.dispatch(t.body) - self.write(")") - - def _alias(self, t): - self.write(t.name) - if t.asname: - self.write(" as "+t.asname) - - def _withitem(self, t): - self.dispatch(t.context_expr) - if t.optional_vars: - self.write(" as ") - self.dispatch(t.optional_vars) - -def roundtrip(filename, outpath): - with open(filename, "rb") as pyfile: - encoding = tokenize.detect_encoding(pyfile.readline)[0] - with open(filename, "r", encoding=encoding) as pyfile: - source = pyfile.read() - tree = compile(source, filename, "exec", ast.PyCF_ONLY_AST) - with open(outpath, "w", encoding=encoding) as output: - Unparser(tree, output) - -def strip_comments_and_docstrings(path): - tmp = path + ".tmp" - if path.endswith(".py"): - roundtrip(path, tmp) - else: - shutil.copy(path, tmp) - return tmp From 795b767b6effffbc32ec07f9024638780bbb3ad4 Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Thu, 4 Apr 2024 22:48:12 +0200 Subject: [PATCH 41/97] add link to the source variable in the alert-message for java/implicit-cast-in-compound-assignment --- java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql | 9 +++++---- .../CWE-190/semmle/tests/InformationLoss.expected | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql b/java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql index d2ff4c24060..4d770b462b8 100644 --- a/java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql +++ b/java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql @@ -27,10 +27,11 @@ class DangerousAssignOpExpr extends AssignOp { predicate problematicCasting(Type t, Expr e) { e.getType().(NumType).widerThan(t) } -from DangerousAssignOpExpr a, Expr e +from DangerousAssignOpExpr a, Expr e, Variable v where e = a.getSource() and - problematicCasting(a.getDest().getType(), e) + problematicCasting(a.getDest().getType(), e) and + v = a.getDest().(VarAccess).getVariable() select a, - "Implicit cast of source type " + e.getType().getName() + " to narrower destination type " + - a.getDest().getType().getName() + "." + "Implicit cast of source $@ to narrower destination type " + a.getDest().getType().getName() + ".", + v, "type " + e.getType().getName() diff --git a/java/ql/test/query-tests/security/CWE-190/semmle/tests/InformationLoss.expected b/java/ql/test/query-tests/security/CWE-190/semmle/tests/InformationLoss.expected index 1c4ed46e136..03f39286a19 100644 --- a/java/ql/test/query-tests/security/CWE-190/semmle/tests/InformationLoss.expected +++ b/java/ql/test/query-tests/security/CWE-190/semmle/tests/InformationLoss.expected @@ -1,2 +1,2 @@ -| Test.java:68:5:68:25 | ...+=... | Implicit cast of source type long to narrower destination type int. | -| Test.java:87:4:87:9 | ...+=... | Implicit cast of source type long to narrower destination type int. | +| Test.java:68:5:68:25 | ...+=... | Implicit cast of source $@ to narrower destination type int. | Test.java:64:4:64:13 | int i | type long | +| Test.java:87:4:87:9 | ...+=... | Implicit cast of source $@ to narrower destination type int. | Test.java:81:4:81:13 | int i | type long | From 8b220cc1b39775c75c2edaef0f2cbacbdc720ff6 Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Fri, 5 Apr 2024 09:21:18 +0200 Subject: [PATCH 42/97] also get the variable for array accesses --- java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql | 8 +++++++- .../CWE-190/semmle/tests/InformationLoss.expected | 1 + .../security/CWE-190/semmle/tests/Test.java | 10 ++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql b/java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql index 4d770b462b8..8a491c33cc3 100644 --- a/java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql +++ b/java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql @@ -27,11 +27,17 @@ class DangerousAssignOpExpr extends AssignOp { predicate problematicCasting(Type t, Expr e) { e.getType().(NumType).widerThan(t) } +Variable getVariable(DangerousAssignOpExpr a) { + result = a.getDest().(VarAccess).getVariable() + or + result = a.getDest().(ArrayAccess).getArray().(VarAccess).getVariable() +} + from DangerousAssignOpExpr a, Expr e, Variable v where e = a.getSource() and problematicCasting(a.getDest().getType(), e) and - v = a.getDest().(VarAccess).getVariable() + v = getVariable(a) select a, "Implicit cast of source $@ to narrower destination type " + a.getDest().getType().getName() + ".", v, "type " + e.getType().getName() diff --git a/java/ql/test/query-tests/security/CWE-190/semmle/tests/InformationLoss.expected b/java/ql/test/query-tests/security/CWE-190/semmle/tests/InformationLoss.expected index 03f39286a19..91bac65629a 100644 --- a/java/ql/test/query-tests/security/CWE-190/semmle/tests/InformationLoss.expected +++ b/java/ql/test/query-tests/security/CWE-190/semmle/tests/InformationLoss.expected @@ -1,2 +1,3 @@ | Test.java:68:5:68:25 | ...+=... | Implicit cast of source $@ to narrower destination type int. | Test.java:64:4:64:13 | int i | type long | | Test.java:87:4:87:9 | ...+=... | Implicit cast of source $@ to narrower destination type int. | Test.java:81:4:81:13 | int i | type long | +| Test.java:289:5:289:30 | ...+=... | Implicit cast of source $@ to narrower destination type int. | Test.java:285:4:285:27 | int[] arr | type long | diff --git a/java/ql/test/query-tests/security/CWE-190/semmle/tests/Test.java b/java/ql/test/query-tests/security/CWE-190/semmle/tests/Test.java index 2935368dccb..6dcfeefd2d1 100644 --- a/java/ql/test/query-tests/security/CWE-190/semmle/tests/Test.java +++ b/java/ql/test/query-tests/security/CWE-190/semmle/tests/Test.java @@ -279,6 +279,16 @@ class Test { // subsequently cast to narrower type int int widenedThenNarrowed = (int) (data2 + 10L); } + + // InformationLoss + { + int[] arr = new int[10]; + while (arr[2] < 1000000) { + // BAD: getLargeNumber is implicitly narrowed to an integer + // which will result in overflows if it is large + arr[2] += getLargeNumber(); + } + } } public static long getLargeNumber() { From ca4f667053399a04eef7f7992ebdb616eda6cf57 Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Fri, 5 Apr 2024 16:18:20 +0200 Subject: [PATCH 43/97] add fallback if I can't easily determine the variable --- .../Likely Bugs/Arithmetic/InformationLoss.ql | 20 ++++++++++++------- .../semmle/tests/InformationLoss.expected | 7 ++++--- .../security/CWE-190/semmle/tests/Test.java | 7 +++++++ 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql b/java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql index 8a491c33cc3..71699fd3a5a 100644 --- a/java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql +++ b/java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql @@ -27,17 +27,23 @@ class DangerousAssignOpExpr extends AssignOp { predicate problematicCasting(Type t, Expr e) { e.getType().(NumType).widerThan(t) } -Variable getVariable(DangerousAssignOpExpr a) { - result = a.getDest().(VarAccess).getVariable() +Variable getVariable(Expr dest) { + result = dest.(VarAccess).getVariable() or - result = a.getDest().(ArrayAccess).getArray().(VarAccess).getVariable() + result = dest.(ArrayAccess).getArray().(VarAccess).getVariable() } -from DangerousAssignOpExpr a, Expr e, Variable v +from DangerousAssignOpExpr a, Expr e, Top v where e = a.getSource() and problematicCasting(a.getDest().getType(), e) and - v = getVariable(a) + ( + v = getVariable(a.getDest()) + or + // fallback, in case we can't easily determine the variable + not exists(getVariable(a.getDest())) and + v = a.getDest() + ) select a, - "Implicit cast of source $@ to narrower destination type " + a.getDest().getType().getName() + ".", - v, "type " + e.getType().getName() + "Implicit cast of $@ to narrower destination type " + a.getDest().getType().getName() + ".", + v, "source type " + e.getType().getName() diff --git a/java/ql/test/query-tests/security/CWE-190/semmle/tests/InformationLoss.expected b/java/ql/test/query-tests/security/CWE-190/semmle/tests/InformationLoss.expected index 91bac65629a..dbda11bff08 100644 --- a/java/ql/test/query-tests/security/CWE-190/semmle/tests/InformationLoss.expected +++ b/java/ql/test/query-tests/security/CWE-190/semmle/tests/InformationLoss.expected @@ -1,3 +1,4 @@ -| Test.java:68:5:68:25 | ...+=... | Implicit cast of source $@ to narrower destination type int. | Test.java:64:4:64:13 | int i | type long | -| Test.java:87:4:87:9 | ...+=... | Implicit cast of source $@ to narrower destination type int. | Test.java:81:4:81:13 | int i | type long | -| Test.java:289:5:289:30 | ...+=... | Implicit cast of source $@ to narrower destination type int. | Test.java:285:4:285:27 | int[] arr | type long | +| Test.java:68:5:68:25 | ...+=... | Implicit cast of $@ to narrower destination type int. | Test.java:64:4:64:13 | int i | source type long | +| Test.java:87:4:87:9 | ...+=... | Implicit cast of $@ to narrower destination type int. | Test.java:81:4:81:13 | int i | source type long | +| Test.java:289:5:289:30 | ...+=... | Implicit cast of $@ to narrower destination type int. | Test.java:285:4:285:27 | int[] arr | source type long | +| Test.java:293:7:293:44 | ...+=... | Implicit cast of $@ to narrower destination type int. | Test.java:293:7:293:24 | ...[...] | source type long | diff --git a/java/ql/test/query-tests/security/CWE-190/semmle/tests/Test.java b/java/ql/test/query-tests/security/CWE-190/semmle/tests/Test.java index 6dcfeefd2d1..f24d16a236c 100644 --- a/java/ql/test/query-tests/security/CWE-190/semmle/tests/Test.java +++ b/java/ql/test/query-tests/security/CWE-190/semmle/tests/Test.java @@ -288,6 +288,9 @@ class Test { // which will result in overflows if it is large arr[2] += getLargeNumber(); } + + // BAD. + getAnIntArray()[0] += getLargeNumber(); } } @@ -295,6 +298,10 @@ class Test { return Long.MAX_VALUE / 2; } + public static int[] getAnIntArray() { + return new int[10]; + } + public static boolean properlyBounded(int i) { return i < Integer.MAX_VALUE; } From 018b066b9550571d0e4ebdd6d936c6d2a1b4be73 Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Mon, 8 Apr 2024 07:15:33 +0200 Subject: [PATCH 44/97] autoformat --- java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql b/java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql index 71699fd3a5a..e8893f3bcc7 100644 --- a/java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql +++ b/java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql @@ -45,5 +45,5 @@ where v = a.getDest() ) select a, - "Implicit cast of $@ to narrower destination type " + a.getDest().getType().getName() + ".", - v, "source type " + e.getType().getName() + "Implicit cast of $@ to narrower destination type " + a.getDest().getType().getName() + ".", v, + "source type " + e.getType().getName() From 9aa85f2d132ce8f0f78e5744d6f5430d7e8d9b27 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Thu, 4 Apr 2024 14:26:13 +0200 Subject: [PATCH 45/97] C#: Validate all nuget feeds to respond in reasonable time --- .../DependencyManager.Nuget.cs | 191 +++++++++++++++--- .../DependencyManager.cs | 9 +- .../DotNet.cs | 17 +- .../EnvironmentVariableNames.cs | 15 ++ .../IDotNet.cs | 1 + .../Semmle.Extraction.Tests/Runtime.cs | 2 + .../Semmle.Util/EnvironmentVariables.cs | 7 + csharp/extractor/Semmle.Util/FileUtils.cs | 5 +- 8 files changed, 208 insertions(+), 39 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs index 8e0be8ab141..cdf4f5dcf4a 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs @@ -2,7 +2,9 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Net.Http; using System.Text.RegularExpressions; +using System.Threading; using System.Threading.Tasks; using Semmle.Util; @@ -14,6 +16,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { try { + var checkNugetFeedResponsiveness = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.CheckNugetFeedResponsiveness); + if (checkNugetFeedResponsiveness && !CheckFeeds(allNonBinaryFiles)) + { + DownloadMissingPackages(allNonBinaryFiles, dllPaths, withNugetConfig: false); + return; + } + using (var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger)) { var count = nuget.InstallPackages(); @@ -139,7 +148,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString())); } - private void DownloadMissingPackages(List allFiles, ISet dllPaths) + private void DownloadMissingPackages(List allFiles, ISet dllPaths, bool withNugetConfig = true) { var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo); var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames(); @@ -172,30 +181,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored"); - - var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray(); - string? nugetConfig = null; - if (nugetConfigs.Length > 1) - { - logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}."); - nugetConfig = allFiles - .SelectRootFiles(sourceDir) - .SelectFileNamesByName("nuget.config") - .FirstOrDefault(); - if (nugetConfig == null) - { - logger.LogInfo("Could not find a top-level nuget.config file."); - } - } - else - { - nugetConfig = nugetConfigs.FirstOrDefault(); - } - - if (nugetConfig != null) - { - logger.LogInfo($"Using nuget.config file {nugetConfig}."); - } + var nugetConfig = withNugetConfig + ? GetNugetConfig(allFiles) + : null; CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString())); @@ -221,6 +209,37 @@ namespace Semmle.Extraction.CSharp.DependencyFetching dllPaths.Add(missingPackageDirectory.DirInfo.FullName); } + private string[] GetAllNugetConfigs(List allFiles) => allFiles.SelectFileNamesByName("nuget.config").ToArray(); + + private string? GetNugetConfig(List allFiles) + { + var nugetConfigs = GetAllNugetConfigs(allFiles); + string? nugetConfig; + if (nugetConfigs.Length > 1) + { + logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}."); + nugetConfig = allFiles + .SelectRootFiles(sourceDir) + .SelectFileNamesByName("nuget.config") + .FirstOrDefault(); + if (nugetConfig == null) + { + logger.LogInfo("Could not find a top-level nuget.config file."); + } + } + else + { + nugetConfig = nugetConfigs.FirstOrDefault(); + } + + if (nugetConfig != null) + { + logger.LogInfo($"Using nuget.config file {nugetConfig}."); + } + + return nugetConfig; + } + private void LogAllUnusedPackages(DependencyContainer dependencies) { var allPackageDirectories = GetAllPackageDirectories(); @@ -279,9 +298,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching .Select(d => Path.GetFileName(d).ToLowerInvariant()); } - [GeneratedRegex(@".*", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)] - private static partial Regex TargetFramework(); - private bool TryRestorePackageManually(string package, string? nugetConfig, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj) { logger.LogInfo($"Restoring package {package}..."); @@ -358,7 +374,126 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } + private static async Task ExecuteGetRequest(string address, HttpClient httpClient, CancellationToken cancellationToken) + { + using var stream = await httpClient.GetStreamAsync(address, cancellationToken); + var buffer = new byte[1024]; + int bytesRead; + while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0) + { + // do nothing + } + } + + private bool IsFeedReachable(string feed) + { + using HttpClient client = new(); + var timeoutSeconds = 1; + var tryCount = 4; + + for (var i = 0; i < tryCount; i++) + { + using var cts = new CancellationTokenSource(); + cts.CancelAfter(timeoutSeconds * 1000); + try + { + ExecuteGetRequest(feed, client, cts.Token).GetAwaiter().GetResult(); + return true; + } + catch (Exception exc) + { + if (exc is TaskCanceledException tce && + tce.CancellationToken == cts.Token && + cts.Token.IsCancellationRequested) + { + logger.LogWarning($"Didn't receive answer from Nuget feed '{feed}' in {timeoutSeconds} seconds."); + timeoutSeconds *= 2; + continue; + } + + // We're only interested in timeouts. + logger.LogWarning($"Querying Nuget feed '{feed}' failed: {exc}"); + return true; + } + } + + logger.LogError($"Didn't receive answer from Nuget feed '{feed}'. Tried it {tryCount} times."); + return false; + } + + private bool CheckFeeds(List allFiles) + { + logger.LogInfo("Checking Nuget feeds..."); + var feeds = GetAllFeeds(allFiles); + + var excludedFeeds = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck) + ?.Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries) + .ToHashSet() ?? []; + + if (excludedFeeds.Count > 0) + { + logger.LogInfo($"Excluded feeds from responsiveness check: {string.Join(", ", excludedFeeds)}"); + } + + var allFeedsReachable = feeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed)); + if (!allFeedsReachable) + { + diagnosticsWriter.AddEntry(new DiagnosticMessage( + Language.CSharp, + "buildless/unreachable-feed", + "Found unreachable Nuget feed in C# analysis with build-mode 'none'", + visibility: new DiagnosticMessage.TspVisibility(statusPage: true, cliSummaryTable: true, telemetry: true), + markdownMessage: "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.", + severity: DiagnosticMessage.TspSeverity.Warning + )); + } + CompilationInfos.Add(("All Nuget feeds reachable", allFeedsReachable ? "1" : "0")); + return allFeedsReachable; + } + + private IEnumerable GetFeeds(string nugetConfig) + { + logger.LogInfo($"Getting Nuget feeds from '{nugetConfig}'..."); + var results = dotnet.GetNugetFeeds(nugetConfig); + var regex = EnabledNugetFeed(); + foreach (var result in results) + { + var match = regex.Match(result); + if (!match.Success) + { + logger.LogError($"Failed to parse feed from '{result}'"); + continue; + } + + var url = match.Groups[1].Value; + if (!url.StartsWith("https://", StringComparison.InvariantCultureIgnoreCase) && + !url.StartsWith("http://", StringComparison.InvariantCultureIgnoreCase)) + { + logger.LogInfo($"Skipping feed '{url}' as it is not a valid URL."); + continue; + } + + yield return url; + } + } + + private HashSet GetAllFeeds(List allFiles) + { + var nugetConfigs = GetAllNugetConfigs(allFiles); + var feeds = nugetConfigs + .SelectMany(nf => GetFeeds(nf)) + .Where(str => !string.IsNullOrWhiteSpace(str)) + .ToHashSet(); + return feeds; + } + + [GeneratedRegex(@".*", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)] + private static partial Regex TargetFramework(); + [GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)] private static partial Regex LegacyNugetPackage(); + + [GeneratedRegex(@"^E (.*)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)] + private static partial Regex EnabledNugetFeed(); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index 04716a64db4..2fc77d004f2 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -20,6 +20,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { private readonly AssemblyCache assemblyCache; private readonly ILogger logger; + private readonly IDiagnosticsWriter diagnosticsWriter; // Only used as a set, but ConcurrentDictionary is the only concurrent set in .NET. private readonly IDictionary usedReferences = new ConcurrentDictionary(); @@ -52,6 +53,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var startTime = DateTime.Now; this.logger = logger; + this.diagnosticsWriter = new DiagnosticsStream(Path.Combine( + Environment.GetEnvironmentVariable(EnvironmentVariableNames.DiagnosticDir) ?? "", + $"dependency-manager-{DateTime.UtcNow:yyyyMMddHHmm}-{Environment.ProcessId}.jsonc")); this.sourceDir = new DirectoryInfo(srcDir); packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "packages")); @@ -177,8 +181,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var frameworkLocations = new HashSet(); var frameworkReferences = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferences); - var frameworkReferencesUseSubfolders = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders); - _ = bool.TryParse(frameworkReferencesUseSubfolders, out var useSubfolders); + var useSubfolders = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders); if (!string.IsNullOrWhiteSpace(frameworkReferences)) { RemoveFrameworkNugetPackages(dllPaths); @@ -740,6 +743,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { Dispose(tempWorkingDirectory, "temporary working"); } + + diagnosticsWriter?.Dispose(); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs index b132d1884f9..c57958845f2 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs @@ -16,12 +16,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching public partial class DotNet : IDotNet { private readonly IDotNetCliInvoker dotnetCliInvoker; + private readonly ILogger logger; private readonly TemporaryDirectory? tempWorkingDirectory; private DotNet(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, TemporaryDirectory? tempWorkingDirectory = null) { this.tempWorkingDirectory = tempWorkingDirectory; this.dotnetCliInvoker = dotnetCliInvoker; + this.logger = logger; Info(); } @@ -89,17 +91,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return dotnetCliInvoker.RunCommand(args); } - public IList GetListedRuntimes() => GetListed("--list-runtimes"); + public IList GetListedRuntimes() => GetResultList("--list-runtimes"); - public IList GetListedSdks() => GetListed("--list-sdks"); + public IList GetListedSdks() => GetResultList("--list-sdks"); - private IList GetListed(string args) + private IList GetResultList(string args) { - if (dotnetCliInvoker.RunCommand(args, out var artifacts)) + if (dotnetCliInvoker.RunCommand(args, out var results)) { - return artifacts; + return results; } - return new List(); + logger.LogWarning($"Running 'dotnet {args}' failed."); + return []; } public bool Exec(string execArgs) @@ -108,6 +111,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return dotnetCliInvoker.RunCommand(args); } + public IList GetNugetFeeds(string nugetConfig) => GetResultList($"nuget list source --format Short --configfile \"{nugetConfig}\""); + // The version number should be kept in sync with the version .NET version used for building the application. public const string LatestDotNetSdkVersion = "8.0.101"; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs index 65a4664e83e..2d36319042a 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs @@ -16,5 +16,20 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// Controls whether to use framework dependencies from subfolders. /// public const string DotnetFrameworkReferencesUseSubfolders = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES_USE_SUBFOLDERS"; + + /// + /// Controls whether to check the responsiveness of NuGet feeds. + /// + public const string CheckNugetFeedResponsiveness = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK"; + + /// + /// Specifies the NuGet feeds to exclude from the responsiveness check. + /// + public const string ExcludedNugetFeedsFromResponsivenessCheck = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_EXCLUDED_FROM_CHECK"; + + /// + /// Specifies the location of the diagnostic directory. + /// + public const string DiagnosticDir = "CODEQL_EXTRACTOR_CSHARP_DIAGNOSTIC_DIR"; } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs index d66135c1644..d97fc7d6441 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs @@ -13,6 +13,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching IList GetListedRuntimes(); IList GetListedSdks(); bool Exec(string execArgs); + IList GetNugetFeeds(string nugetConfig); } public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false); diff --git a/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs b/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs index 17bc477bde8..2daf8244d97 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs @@ -26,6 +26,8 @@ namespace Semmle.Extraction.Tests public IList GetListedSdks() => sdks; public bool Exec(string execArgs) => true; + + public IList GetNugetFeeds(string nugetConfig) => []; } public class RuntimeTests diff --git a/csharp/extractor/Semmle.Util/EnvironmentVariables.cs b/csharp/extractor/Semmle.Util/EnvironmentVariables.cs index c96aa16357c..72ba9224669 100644 --- a/csharp/extractor/Semmle.Util/EnvironmentVariables.cs +++ b/csharp/extractor/Semmle.Util/EnvironmentVariables.cs @@ -27,5 +27,12 @@ namespace Semmle.Util } return threads; } + + public static bool GetBoolean(string name) + { + var env = Environment.GetEnvironmentVariable(name); + var _ = bool.TryParse(env, out var value); + return value; + } } } diff --git a/csharp/extractor/Semmle.Util/FileUtils.cs b/csharp/extractor/Semmle.Util/FileUtils.cs index 094c4da3338..4a22877e3c1 100644 --- a/csharp/extractor/Semmle.Util/FileUtils.cs +++ b/csharp/extractor/Semmle.Util/FileUtils.cs @@ -102,8 +102,7 @@ namespace Semmle.Util private static async Task DownloadFileAsync(string address, string filename) { using var httpClient = new HttpClient(); - using var request = new HttpRequestMessage(HttpMethod.Get, address); - using var contentStream = await (await httpClient.SendAsync(request)).Content.ReadAsStreamAsync(); + using var contentStream = await httpClient.GetStreamAsync(address); using var stream = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None, 4096, true); await contentStream.CopyToAsync(stream); } @@ -112,7 +111,7 @@ namespace Semmle.Util /// Downloads the file at to . /// public static void DownloadFile(string address, string fileName) => - DownloadFileAsync(address, fileName).Wait(); + DownloadFileAsync(address, fileName).GetAwaiter().GetResult(); public static string NestPaths(ILogger logger, string? outerpath, string innerpath) { From 7051db5e1c5df825f50a18f805680a8231a17fbe Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Fri, 5 Apr 2024 14:20:42 +0200 Subject: [PATCH 46/97] Fix code review findings --- .../DependencyManager.Nuget.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs index cdf4f5dcf4a..8f8e932279c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs @@ -387,6 +387,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private bool IsFeedReachable(string feed) { + logger.LogInfo($"Checking if Nuget feed '{feed}' is reachable..."); using HttpClient client = new(); var timeoutSeconds = 1; var tryCount = 4; @@ -432,7 +433,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (excludedFeeds.Count > 0) { - logger.LogInfo($"Excluded feeds from responsiveness check: {string.Join(", ", excludedFeeds)}"); + logger.LogInfo($"Excluded feeds from responsiveness check: {string.Join(", ", excludedFeeds.OrderBy(f => f))}"); } var allFeedsReachable = feeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed)); @@ -481,9 +482,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { var nugetConfigs = GetAllNugetConfigs(allFiles); var feeds = nugetConfigs - .SelectMany(nf => GetFeeds(nf)) + .SelectMany(GetFeeds) .Where(str => !string.IsNullOrWhiteSpace(str)) .ToHashSet(); + logger.LogInfo($"Found Nuget feeds in nuget.config files: {string.Join(", ", feeds.OrderBy(f => f))}"); return feeds; } @@ -493,7 +495,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching [GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)] private static partial Regex LegacyNugetPackage(); - [GeneratedRegex(@"^E (.*)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)] + [GeneratedRegex(@"^E\s(.*)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)] private static partial Regex EnabledNugetFeed(); } } From 6a5520c85d78bd07486003e7555b725715f64af3 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Fri, 5 Apr 2024 15:12:15 +0200 Subject: [PATCH 47/97] Add integration test for unreachable nuget feeds --- .../DependencyManager.Nuget.cs | 21 ++++++---- .../EnvironmentVariableNames.cs | 12 +++++- .../Assemblies.expected | 1 + .../Assemblies.ql | 11 +++++ .../CompilationInfo.expected | 13 ++++++ .../CompilationInfo.ql | 15 +++++++ .../diagnostics.expected | 42 +++++++++++++++++++ .../proj/Program.cs | 6 +++ .../proj/nuget.config | 8 ++++ .../proj/proj.csproj | 16 +++++++ .../standalone.sln | 19 +++++++++ .../test.py | 10 +++++ 12 files changed, 165 insertions(+), 9 deletions(-) create mode 100644 csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/Assemblies.expected create mode 100644 csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/Assemblies.ql create mode 100644 csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.expected create mode 100644 csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.ql create mode 100644 csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/diagnostics.expected create mode 100644 csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/Program.cs create mode 100644 csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/nuget.config create mode 100644 csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/proj.csproj create mode 100644 csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/standalone.sln create mode 100644 csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/test.py diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs index 8f8e932279c..c624c8113f6 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs @@ -389,13 +389,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { logger.LogInfo($"Checking if Nuget feed '{feed}' is reachable..."); using HttpClient client = new(); - var timeoutSeconds = 1; - var tryCount = 4; + int timeoutMilliSeconds = int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessInitialTimeout), out timeoutMilliSeconds) + ? timeoutMilliSeconds + : 1000; + int tryCount = int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessRequestCount), out tryCount) + ? tryCount + : 4; for (var i = 0; i < tryCount; i++) { using var cts = new CancellationTokenSource(); - cts.CancelAfter(timeoutSeconds * 1000); + cts.CancelAfter(timeoutMilliSeconds); try { ExecuteGetRequest(feed, client, cts.Token).GetAwaiter().GetResult(); @@ -407,8 +411,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching tce.CancellationToken == cts.Token && cts.Token.IsCancellationRequested) { - logger.LogWarning($"Didn't receive answer from Nuget feed '{feed}' in {timeoutSeconds} seconds."); - timeoutSeconds *= 2; + logger.LogWarning($"Didn't receive answer from Nuget feed '{feed}' in {timeoutMilliSeconds}ms."); + timeoutMilliSeconds *= 2; continue; } @@ -418,7 +422,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - logger.LogError($"Didn't receive answer from Nuget feed '{feed}'. Tried it {tryCount} times."); + logger.LogWarning($"Didn't receive answer from Nuget feed '{feed}'. Tried it {tryCount} times."); return false; } @@ -428,17 +432,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var feeds = GetAllFeeds(allFiles); var excludedFeeds = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck) - ?.Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries) + ?.Split(" ", StringSplitOptions.RemoveEmptyEntries) .ToHashSet() ?? []; if (excludedFeeds.Count > 0) { - logger.LogInfo($"Excluded feeds from responsiveness check: {string.Join(", ", excludedFeeds.OrderBy(f => f))}"); + logger.LogInfo($"Excluded Nuget feeds from responsiveness check: {string.Join(", ", excludedFeeds.OrderBy(f => f))}"); } var allFeedsReachable = feeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed)); if (!allFeedsReachable) { + logger.LogWarning("Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis."); diagnosticsWriter.AddEntry(new DiagnosticMessage( Language.CSharp, "buildless/unreachable-feed", diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs index 2d36319042a..ddec97d3dde 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs @@ -25,7 +25,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// /// Specifies the NuGet feeds to exclude from the responsiveness check. /// - public const string ExcludedNugetFeedsFromResponsivenessCheck = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_EXCLUDED_FROM_CHECK"; + public const string ExcludedNugetFeedsFromResponsivenessCheck = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_EXCLUDED"; + + /// + /// Specifies the timeout for the initial check of NuGet feeds responsiveness. + /// + public const string NugetFeedResponsivenessInitialTimeout = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT"; + + /// + /// Specifies how many requests to make to the NuGet feed to check its responsiveness. + /// + public const string NugetFeedResponsivenessRequestCount = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT"; /// /// Specifies the location of the diagnostic directory. diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/Assemblies.expected b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/Assemblies.expected new file mode 100644 index 00000000000..2a530060edb --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/Assemblies.expected @@ -0,0 +1 @@ +| [...]/newtonsoft.json/13.0.3/lib/net6.0/Newtonsoft.Json.dll | diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/Assemblies.ql b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/Assemblies.ql new file mode 100644 index 00000000000..79cf92de791 --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/Assemblies.ql @@ -0,0 +1,11 @@ +import csharp + +private string getPath(Assembly a) { + not a.getCompilation().getOutputAssembly() = a and + exists(string s | s = a.getFile().getAbsolutePath() | + result = "[...]/" + s.substring(s.indexOf("newtonsoft.json"), s.length()) + ) +} + +from Assembly a +select getPath(a) diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.expected b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.expected new file mode 100644 index 00000000000..394fb93a259 --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.expected @@ -0,0 +1,13 @@ +| All Nuget feeds reachable | 0.0 | +| Fallback nuget restore | 1.0 | +| Project files on filesystem | 1.0 | +| Resolved assembly conflicts | 7.0 | +| Restored .NET framework variants | 0.0 | +| Solution files on filesystem | 1.0 | +| Source files generated | 0.0 | +| Source files on filesystem | 1.0 | +| Successfully ran fallback nuget restore | 1.0 | +| Unresolved references | 0.0 | +| UseWPF set | 0.0 | +| UseWindowsForms set | 0.0 | +| WebView extraction enabled | 1.0 | diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.ql b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.ql new file mode 100644 index 00000000000..073ffe3b224 --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.ql @@ -0,0 +1,15 @@ +import csharp +import semmle.code.csharp.commons.Diagnostics + +query predicate compilationInfo(string key, float value) { + key != "Resolved references" and + not key.matches("Compiler diagnostic count for%") and + exists(Compilation c, string infoKey, string infoValue | infoValue = c.getInfo(infoKey) | + key = infoKey and + value = infoValue.toFloat() + or + not exists(infoValue.toFloat()) and + key = infoKey + ": " + infoValue and + value = 1 + ) +} diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/diagnostics.expected b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/diagnostics.expected new file mode 100644 index 00000000000..5f298cd3a11 --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/diagnostics.expected @@ -0,0 +1,42 @@ +{ + "markdownMessage": "C# analysis with build-mode 'none' completed.", + "severity": "unknown", + "source": { + "extractorName": "csharp", + "id": "csharp/autobuilder/buildless/complete", + "name": "C# analysis with build-mode 'none' completed" + }, + "visibility": { + "cliSummaryTable": true, + "statusPage": false, + "telemetry": true + } +} +{ + "markdownMessage": "C# with build-mode set to 'none'. This means that all C# source in the working directory will be scanned, with build tools, such as Nuget and Dotnet CLIs, only contributing information about external dependencies.", + "severity": "note", + "source": { + "extractorName": "csharp", + "id": "csharp/autobuilder/buildless/mode-active", + "name": "C# with build-mode set to 'none'" + }, + "visibility": { + "cliSummaryTable": true, + "statusPage": true, + "telemetry": true + } +} +{ + "markdownMessage": "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.", + "severity": "warning", + "source": { + "extractorName": "csharp", + "id": "csharp/autobuilder/buildless/unreachable-feed", + "name": "Found unreachable Nuget feed in C# analysis with build-mode 'none'" + }, + "visibility": { + "cliSummaryTable": true, + "statusPage": true, + "telemetry": true + } +} diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/Program.cs b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/Program.cs new file mode 100644 index 00000000000..39a9e95bb6e --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/Program.cs @@ -0,0 +1,6 @@ +class Program +{ + static void Main(string[] args) + { + } +} \ No newline at end of file diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/nuget.config b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/nuget.config new file mode 100644 index 00000000000..11d134c7289 --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/nuget.config @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/proj.csproj b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/proj.csproj new file mode 100644 index 00000000000..cef71796352 --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/proj.csproj @@ -0,0 +1,16 @@ + + + + Exe + net8.0 + + + + + + + + + + + diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/standalone.sln b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/standalone.sln new file mode 100644 index 00000000000..493ab54b59a --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/standalone.sln @@ -0,0 +1,19 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.002.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "proj", "proj\proj.csproj", "{6ED00460-7666-4AE9-A405-4B6C8B02279A}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {4ED55A1C-066C-43DF-B32E-7EAA035985EE} + EndGlobalSection +EndGlobal diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/test.py b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/test.py new file mode 100644 index 00000000000..b48382a66ce --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/test.py @@ -0,0 +1,10 @@ +from create_database_utils import * +from diagnostics_test_utils import * +import os + +os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK"] = "true" # Enable NuGet feed check +os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT"] = "1" # 1ms, the GET request should fail with such short timeout +os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT"] = "1" # Limit the count of checks to 1 +os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_EXCLUDED"] = "https://abc.de:8000/packages/" # Exclude this feed from check +run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"]) +check_diagnostics() \ No newline at end of file From 95896bc95fe23dfdbad68320f9524c832de118a9 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Fri, 5 Apr 2024 15:57:07 +0200 Subject: [PATCH 48/97] Make sure diagnostic directory exists --- .../DependencyManager.cs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index 2fc77d004f2..8d63d7f42b2 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -53,8 +53,24 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var startTime = DateTime.Now; this.logger = logger; + + var diagDirEnv = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DiagnosticDir); + if (!string.IsNullOrWhiteSpace(diagDirEnv) && + !Directory.Exists(diagDirEnv)) + { + try + { + Directory.CreateDirectory(diagDirEnv); + } + catch (Exception e) + { + logger.LogError($"Failed to create diagnostic directory {diagDirEnv}: {e.Message}"); + diagDirEnv = null; + } + } + this.diagnosticsWriter = new DiagnosticsStream(Path.Combine( - Environment.GetEnvironmentVariable(EnvironmentVariableNames.DiagnosticDir) ?? "", + diagDirEnv ?? "", $"dependency-manager-{DateTime.UtcNow:yyyyMMddHHmm}-{Environment.ProcessId}.jsonc")); this.sourceDir = new DirectoryInfo(srcDir); From d7f8b9615857ba68243327e12c9af04a68660efe Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Mon, 8 Apr 2024 13:14:51 +0200 Subject: [PATCH 49/97] Improve logging --- .../DependencyManager.Nuget.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs index c624c8113f6..1f3f44425d5 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs @@ -490,7 +490,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching .SelectMany(GetFeeds) .Where(str => !string.IsNullOrWhiteSpace(str)) .ToHashSet(); - logger.LogInfo($"Found Nuget feeds in nuget.config files: {string.Join(", ", feeds.OrderBy(f => f))}"); + + if (feeds.Count > 0) + { + logger.LogInfo($"Found {feeds.Count} Nuget feeds in nuget.config files: {string.Join(", ", feeds.OrderBy(f => f))}"); + } + else + { + logger.LogDebug("No Nuget feeds found in nuget.config files."); + } return feeds; } From 2fb9c2db6fc53c8adf1ce46b3d084bc4a6565b0c Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Tue, 12 Mar 2024 15:47:15 +0100 Subject: [PATCH 50/97] C#: Remove deprecated qualifiedName predicates. --- csharp/ql/lib/semmle/code/csharp/Element.qll | 29 -------------- csharp/ql/lib/semmle/code/csharp/Member.qll | 38 ------------------- .../ql/lib/semmle/code/csharp/Namespace.qll | 10 ----- 3 files changed, 77 deletions(-) diff --git a/csharp/ql/lib/semmle/code/csharp/Element.qll b/csharp/ql/lib/semmle/code/csharp/Element.qll index 2c69912c993..ccfd1a27925 100644 --- a/csharp/ql/lib/semmle/code/csharp/Element.qll +++ b/csharp/ql/lib/semmle/code/csharp/Element.qll @@ -101,25 +101,6 @@ class NamedElement extends Element, @named_element { /** Holds if this element has name 'name'. */ final predicate hasName(string name) { name = this.getName() } - /** - * Gets the fully qualified name of this element, for example the - * fully qualified name of `M` on line 3 is `N.C.M` in - * - * ```csharp - * namespace N { - * class C { - * void M(int i, string s) { } - * } - * } - * ``` - */ - cached - deprecated final string getQualifiedName() { - exists(string qualifier, string name | this.hasQualifiedName(qualifier, name) | - if qualifier = "" then result = name else result = qualifier + "." + name - ) - } - /** * Gets the fully qualified name of this element, for example the * fully qualified name of `M` on line 3 is `N.C.M` in @@ -142,16 +123,6 @@ class NamedElement extends Element, @named_element { ) } - /** - * DEPRECATED: Use `hasFullyQualifiedName` instead. - * - * Holds if this element has the qualified name `qualifier`.`name`. - */ - cached - deprecated predicate hasQualifiedName(string qualifier, string name) { - qualifier = "" and name = this.getName() - } - /** Holds if this element has the fully qualified name `qualifier`.`name`. */ cached predicate hasFullyQualifiedName(string qualifier, string name) { diff --git a/csharp/ql/lib/semmle/code/csharp/Member.qll b/csharp/ql/lib/semmle/code/csharp/Member.qll index 1be091170e3..a2758f03942 100644 --- a/csharp/ql/lib/semmle/code/csharp/Member.qll +++ b/csharp/ql/lib/semmle/code/csharp/Member.qll @@ -71,37 +71,10 @@ class Declaration extends NamedElement, @declaration { override string toString() { result = this.getName() } - deprecated override predicate hasQualifiedName(string qualifier, string name) { - QualifiedName::hasQualifiedName(this, qualifier, name) - } - override predicate hasFullyQualifiedName(string qualifier, string name) { QualifiedName::hasQualifiedName(this, qualifier, name) } - /** - * DEPRECATED: Use `getFullyQualifiedNameWithTypes` instead. - * - * Gets the fully qualified name of this declaration, including types, for example - * the fully qualified name with types of `M` on line 3 is `N.C.M(int, string)` in - * - * ```csharp - * namespace N { - * class C { - * void M(int i, string s) { } - * } - * } - * ``` - */ - deprecated string getQualifiedNameWithTypes() { - exists(string qual | - qual = this.getDeclaringType().getQualifiedName() and - if this instanceof NestedType - then result = qual + "+" + this.toStringWithTypes() - else result = qual + "." + this.toStringWithTypes() - ) - } - /** * Gets the fully qualified name of this declaration, including types, for example * the fully qualified name with types of `M` on line 3 is `N.C.M(int, string)` in @@ -263,17 +236,6 @@ class Member extends Modifiable, @member { /** Gets an access to this member. */ MemberAccess getAnAccess() { result.getTarget() = this } - /** - * DEPRECATED: Use `hasFullyQualifiedName` instead. - * - * Holds if this member has name `name` and is defined in type `type` - * with namespace `namespace`. - */ - cached - deprecated final predicate hasQualifiedName(string namespace, string type, string name) { - QualifiedName::hasQualifiedName(this, namespace, type, name) - } - /** * Holds if this member has name `name` and is defined in type `type` * with namespace `namespace`. diff --git a/csharp/ql/lib/semmle/code/csharp/Namespace.qll b/csharp/ql/lib/semmle/code/csharp/Namespace.qll index 002ce444b3f..383fe8ab2a8 100644 --- a/csharp/ql/lib/semmle/code/csharp/Namespace.qll +++ b/csharp/ql/lib/semmle/code/csharp/Namespace.qll @@ -38,16 +38,6 @@ class Namespace extends TypeContainer, Declaration, @namespace { parent_namespace(result, this) } - /** - * Holds if this namespace has the qualified name `qualifier`.`name`. - * - * For example if the qualified name is `System.Collections.Generic`, then - * `qualifier`=`System.Collections` and `name`=`Generic`. - */ - deprecated override predicate hasQualifiedName(string qualifier, string name) { - namespaceHasQualifiedName(this, qualifier, name) - } - /** * Holds if this namespace has the qualified name `qualifier`.`name`. * From 8fa91914349bf21e52fabfde00c68f13b28c8a63 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Wed, 13 Mar 2024 10:12:22 +0100 Subject: [PATCH 51/97] C#: Deprecate the getFullyQualifiedName predicate. --- .../diag_recursive_generics/Types.ql | 7 ++++--- csharp/ql/lib/semmle/code/csharp/Element.qll | 4 +++- csharp/ql/lib/semmle/code/csharp/Member.qll | 9 +++++---- csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll | 8 +++++++- .../csharp/dataflow/internal/FlowSummaryImpl.qll | 15 ++++++++++++--- .../security/dataflow/ExternalAPIsQuery.qll | 6 +++--- .../library-tests/constructors/Destructors1.ql | 2 +- 7 files changed, 35 insertions(+), 16 deletions(-) diff --git a/csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql b/csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql index d59c60ec802..fff011dcbd5 100644 --- a/csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql +++ b/csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql @@ -1,5 +1,6 @@ import csharp +import semmle.code.csharp.commons.QualifiedName -from Class c -where c.fromSource() -select c, c.getBaseClass().getFullyQualifiedName() +from Class c, string qualifier, string name +where c.fromSource() and c.getBaseClass().hasFullyQualifiedName(qualifier, name) +select c, getQualifiedName(qualifier, name) diff --git a/csharp/ql/lib/semmle/code/csharp/Element.qll b/csharp/ql/lib/semmle/code/csharp/Element.qll index ccfd1a27925..38176acfd92 100644 --- a/csharp/ql/lib/semmle/code/csharp/Element.qll +++ b/csharp/ql/lib/semmle/code/csharp/Element.qll @@ -102,6 +102,8 @@ class NamedElement extends Element, @named_element { final predicate hasName(string name) { name = this.getName() } /** + * DEPRECATED: Use `hasFullyQualifiedName` instead. + * * Gets the fully qualified name of this element, for example the * fully qualified name of `M` on line 3 is `N.C.M` in * @@ -117,7 +119,7 @@ class NamedElement extends Element, @named_element { * ``System.Collections.Generic.IList`1``. */ cached - final string getFullyQualifiedName() { + deprecated final string getFullyQualifiedName() { exists(string qualifier, string name | this.hasFullyQualifiedName(qualifier, name) | if qualifier = "" then result = name else result = qualifier + "." + name ) diff --git a/csharp/ql/lib/semmle/code/csharp/Member.qll b/csharp/ql/lib/semmle/code/csharp/Member.qll index a2758f03942..99bf8eaa4f0 100644 --- a/csharp/ql/lib/semmle/code/csharp/Member.qll +++ b/csharp/ql/lib/semmle/code/csharp/Member.qll @@ -88,11 +88,12 @@ class Declaration extends NamedElement, @declaration { * ``` */ string getFullyQualifiedNameWithTypes() { - exists(string qual | - qual = this.getDeclaringType().getFullyQualifiedName() and + exists(string fullqual, string qual, string name | + this.getDeclaringType().hasFullyQualifiedName(qual, name) and + fullqual = getQualifiedName(qual, name) and if this instanceof NestedType - then result = qual + "+" + this.toStringWithTypes() - else result = qual + "." + this.toStringWithTypes() + then result = fullqual + "+" + this.toStringWithTypes() + else result = fullqual + "." + this.toStringWithTypes() ) } diff --git a/csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll b/csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll index 673bd1a5638..9cd33ea260f 100644 --- a/csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll +++ b/csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll @@ -3,6 +3,7 @@ */ import csharp +private import semmle.code.csharp.commons.QualifiedName /** * Provides classes for working with static single assignment (SSA) form. @@ -120,7 +121,12 @@ module Ssa { result = prefix + "." + this.getAssignable() | if f.(Modifiable).isStatic() - then prefix = f.getDeclaringType().getFullyQualifiedName() + then + exists(string qualifier, string name | + f.getDeclaringType().hasFullyQualifiedName(qualifier, name) + | + prefix = getQualifiedName(qualifier, name) + ) else prefix = "this" ) } diff --git a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll index 96f45e77655..82c2a7dcd3d 100644 --- a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll +++ b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll @@ -3,6 +3,7 @@ */ private import csharp +private import semmle.code.csharp.commons.QualifiedName private import semmle.code.csharp.frameworks.system.linq.Expressions private import codeql.dataflow.internal.FlowSummaryImpl private import codeql.dataflow.internal.AccessPathSyntax as AccessPath @@ -42,10 +43,18 @@ module Input implements InputSig string encodeContent(ContentSet c, string arg) { c = TElementContent() and result = "Element" and arg = "" or - exists(Field f | c = TFieldContent(f) and result = "Field" and arg = f.getFullyQualifiedName()) + exists(Field f, string qualifier, string name | + c = TFieldContent(f) and + f.hasFullyQualifiedName(qualifier, name) and + arg = getQualifiedName(qualifier, name) and + result = "Field" + ) or - exists(Property p | - c = TPropertyContent(p) and result = "Property" and arg = p.getFullyQualifiedName() + exists(Property p, string qualifier, string name | + c = TPropertyContent(p) and + p.hasFullyQualifiedName(qualifier, name) and + arg = getQualifiedName(qualifier, name) and + result = "Property" ) or exists(SyntheticField f | diff --git a/csharp/ql/lib/semmle/code/csharp/security/dataflow/ExternalAPIsQuery.qll b/csharp/ql/lib/semmle/code/csharp/security/dataflow/ExternalAPIsQuery.qll index 3075fe53a87..a0d0ada957a 100644 --- a/csharp/ql/lib/semmle/code/csharp/security/dataflow/ExternalAPIsQuery.qll +++ b/csharp/ql/lib/semmle/code/csharp/security/dataflow/ExternalAPIsQuery.qll @@ -139,13 +139,13 @@ class ExternalApiUsedWithUntrustedData extends TExternalApi { /** Gets a textual representation of this element. */ string toString() { - exists(Callable m, int index, string indexString | + exists(Callable m, int index, string indexString, string qualifier, string name | if index = -1 then indexString = "qualifier" else indexString = "param " + index | this = TExternalApiParameter(m, index) and + m.getDeclaringType().hasFullyQualifiedName(qualifier, name) and result = - m.getDeclaringType().getFullyQualifiedName() + "." + m.toStringWithTypes() + " [" + - indexString + "]" + getQualifiedName(qualifier, name) + "." + m.toStringWithTypes() + " [" + indexString + "]" ) } } diff --git a/csharp/ql/test/library-tests/constructors/Destructors1.ql b/csharp/ql/test/library-tests/constructors/Destructors1.ql index 368980a290c..673a0e941f0 100644 --- a/csharp/ql/test/library-tests/constructors/Destructors1.ql +++ b/csharp/ql/test/library-tests/constructors/Destructors1.ql @@ -10,4 +10,4 @@ where c.getDeclaringType().hasFullyQualifiedName(qualifier, name) and qualifier = "Constructors" and name = "Class" -select c, c.getDeclaringType().getFullyQualifiedName() +select c, getQualifiedName(qualifier, name) From b677e89f35fd0f0ae322f0191e342520ab0bc795 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Wed, 13 Mar 2024 10:23:24 +0100 Subject: [PATCH 52/97] C#: Deprecate getFullyQualifiedNameWithTypes. --- csharp/ql/lib/semmle/code/csharp/Member.qll | 2 +- .../code/csharp/commons/QualifiedName.qll | 25 +++++++++++++++++++ .../dispatch/GetADynamicTarget.ql | 3 ++- .../frameworks/system/Dispose/Dispose.ql | 3 ++- .../frameworks/system/Equals/Equals.ql | 3 ++- .../test/library-tests/generics/Generics.ql | 12 ++++----- .../library-tests/overrides/Overrides22.ql | 3 ++- .../library-tests/unification/Unification.ql | 5 ++-- 8 files changed, 43 insertions(+), 13 deletions(-) diff --git a/csharp/ql/lib/semmle/code/csharp/Member.qll b/csharp/ql/lib/semmle/code/csharp/Member.qll index 99bf8eaa4f0..3427d4ea089 100644 --- a/csharp/ql/lib/semmle/code/csharp/Member.qll +++ b/csharp/ql/lib/semmle/code/csharp/Member.qll @@ -87,7 +87,7 @@ class Declaration extends NamedElement, @declaration { * } * ``` */ - string getFullyQualifiedNameWithTypes() { + deprecated string getFullyQualifiedNameWithTypes() { exists(string fullqual, string qual, string name | this.getDeclaringType().hasFullyQualifiedName(qual, name) and fullqual = getQualifiedName(qual, name) and diff --git a/csharp/ql/lib/semmle/code/csharp/commons/QualifiedName.qll b/csharp/ql/lib/semmle/code/csharp/commons/QualifiedName.qll index eba0fb10c7c..417dc137027 100644 --- a/csharp/ql/lib/semmle/code/csharp/commons/QualifiedName.qll +++ b/csharp/ql/lib/semmle/code/csharp/commons/QualifiedName.qll @@ -219,3 +219,28 @@ predicate splitQualifiedName(string qualifiedName, string qualifier, string name name = qualifiedName ) } + +/** + * INTERNAL: Do not use. + * + * Gets the fully qualified name of this declaration, including types, for example + * the fully qualified name with types of `M` on line 3 is `N.C.M(int, string)` in + * + * ```csharp + * namespace N { + * class C { + * void M(int i, string s) { } + * } + * } + * ``` + */ +bindingset[d] +string getFullyQualifiedNameWithTypes(Declaration d) { + exists(string fullqual, string qual, string name | + d.getDeclaringType().hasFullyQualifiedName(qual, name) and + fullqual = getQualifiedName(qual, name) and + if d instanceof NestedType + then result = fullqual + "+" + d.toStringWithTypes() + else result = fullqual + "." + d.toStringWithTypes() + ) +} diff --git a/csharp/ql/test/library-tests/dispatch/GetADynamicTarget.ql b/csharp/ql/test/library-tests/dispatch/GetADynamicTarget.ql index cbde2d43ab8..b4c94a0b507 100644 --- a/csharp/ql/test/library-tests/dispatch/GetADynamicTarget.ql +++ b/csharp/ql/test/library-tests/dispatch/GetADynamicTarget.ql @@ -1,8 +1,9 @@ import csharp +import semmle.code.csharp.commons.QualifiedName import semmle.code.csharp.dispatch.Dispatch from DispatchCall call, Callable callable where callable = call.getADynamicTarget() and callable.fromSource() -select call, callable.getFullyQualifiedNameWithTypes() +select call, getFullyQualifiedNameWithTypes(callable) diff --git a/csharp/ql/test/library-tests/frameworks/system/Dispose/Dispose.ql b/csharp/ql/test/library-tests/frameworks/system/Dispose/Dispose.ql index 943245fa0a2..5fa2f337fcf 100644 --- a/csharp/ql/test/library-tests/frameworks/system/Dispose/Dispose.ql +++ b/csharp/ql/test/library-tests/frameworks/system/Dispose/Dispose.ql @@ -1,4 +1,5 @@ import csharp +import semmle.code.csharp.commons.QualifiedName import semmle.code.csharp.frameworks.System from ValueOrRefType t, Method m, boolean b @@ -6,4 +7,4 @@ where t.fromSource() and m = getInvokedDisposeMethod(t) and if implementsDispose(t) then b = true else b = false -select t, m.getFullyQualifiedNameWithTypes(), b +select t, getFullyQualifiedNameWithTypes(m), b diff --git a/csharp/ql/test/library-tests/frameworks/system/Equals/Equals.ql b/csharp/ql/test/library-tests/frameworks/system/Equals/Equals.ql index 91c04791ef3..e6dc4ae7549 100644 --- a/csharp/ql/test/library-tests/frameworks/system/Equals/Equals.ql +++ b/csharp/ql/test/library-tests/frameworks/system/Equals/Equals.ql @@ -1,4 +1,5 @@ import csharp +import semmle.code.csharp.commons.QualifiedName import semmle.code.csharp.frameworks.System from ValueOrRefType t, Method m, boolean b @@ -6,4 +7,4 @@ where t.fromSource() and m = getInvokedEqualsMethod(t) and if implementsEquals(t) then b = true else b = false -select t, m.getFullyQualifiedNameWithTypes(), b +select t, getFullyQualifiedNameWithTypes(m), b diff --git a/csharp/ql/test/library-tests/generics/Generics.ql b/csharp/ql/test/library-tests/generics/Generics.ql index 2f3aff0fd58..71e21a5815e 100644 --- a/csharp/ql/test/library-tests/generics/Generics.ql +++ b/csharp/ql/test/library-tests/generics/Generics.ql @@ -231,18 +231,18 @@ query predicate test27(ConstructedType ct, UnboundGenericType ugt, UnboundGeneri query predicate test28(UnboundGeneric ug, string s) { ug.fromSource() and - s = ug.getFullyQualifiedNameWithTypes() + s = getFullyQualifiedNameWithTypes(ug) } query predicate test29(ConstructedGeneric cg, string s) { cg.fromSource() and - s = cg.getFullyQualifiedNameWithTypes() + s = getFullyQualifiedNameWithTypes(cg) } query predicate test30(Declaration d, string s) { d.fromSource() and d instanceof @generic and - s = d.getFullyQualifiedNameWithTypes() and + s = getFullyQualifiedNameWithTypes(d) and d != d.getUnboundDeclaration() and not d instanceof Generic } @@ -263,7 +263,7 @@ query predicate test33(ConstructedMethod cm, string s1, string s2) { exists(string namespace, string type, string name | cm.hasFullyQualifiedName(namespace, type, name) and s1 = getQualifiedName(namespace, type, name) ) and - cm.getFullyQualifiedNameWithTypes() = s2 + getFullyQualifiedNameWithTypes(cm) = s2 } query predicate test34(UnboundGeneric ug, string s1, string s2) { @@ -271,7 +271,7 @@ query predicate test34(UnboundGeneric ug, string s1, string s2) { exists(string qualifier, string name | ug.hasFullyQualifiedName(qualifier, name) and s1 = getQualifiedName(qualifier, name) ) and - ug.getFullyQualifiedNameWithTypes() = s2 + getFullyQualifiedNameWithTypes(ug) = s2 } query predicate test35(UnboundGenericMethod gm, string s1, string s2) { @@ -279,5 +279,5 @@ query predicate test35(UnboundGenericMethod gm, string s1, string s2) { exists(string namespace, string type, string name | gm.hasFullyQualifiedName(namespace, type, name) and s1 = getQualifiedName(namespace, type, name) ) and - gm.getFullyQualifiedNameWithTypes() = s2 + getFullyQualifiedNameWithTypes(gm) = s2 } diff --git a/csharp/ql/test/library-tests/overrides/Overrides22.ql b/csharp/ql/test/library-tests/overrides/Overrides22.ql index d2c5a9e4336..d6300d49ecd 100644 --- a/csharp/ql/test/library-tests/overrides/Overrides22.ql +++ b/csharp/ql/test/library-tests/overrides/Overrides22.ql @@ -1,4 +1,5 @@ import csharp +import semmle.code.csharp.commons.QualifiedName from Overridable v1, Overridable v2, string kind where @@ -9,4 +10,4 @@ where ) and v1.fromSource() and v2.fromSource() -select v1.getFullyQualifiedNameWithTypes(), v2.getFullyQualifiedNameWithTypes(), kind +select getFullyQualifiedNameWithTypes(v1), getFullyQualifiedNameWithTypes(v2), kind diff --git a/csharp/ql/test/library-tests/unification/Unification.ql b/csharp/ql/test/library-tests/unification/Unification.ql index 10c5e520921..f8c6c15377d 100644 --- a/csharp/ql/test/library-tests/unification/Unification.ql +++ b/csharp/ql/test/library-tests/unification/Unification.ql @@ -1,3 +1,4 @@ +import semmle.code.csharp.commons.QualifiedName import semmle.code.csharp.Unification class InterestingType extends @type { @@ -7,9 +8,9 @@ class InterestingType extends @type { } string toString() { - result = this.(Type).getFullyQualifiedNameWithTypes() + result = getFullyQualifiedNameWithTypes(this.(Type)) or - not exists(this.(Type).getFullyQualifiedNameWithTypes()) and + not exists(getFullyQualifiedNameWithTypes(this.(Type))) and result = this.(Type).toStringWithTypes() } From 8fbfafc1d7a18356d4577280b813a5bcfec768d0 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Wed, 13 Mar 2024 16:27:01 +0100 Subject: [PATCH 53/97] C#: Dont cache the deprecated getFullyQualifiedName predicate. --- csharp/ql/lib/semmle/code/csharp/Element.qll | 1 - 1 file changed, 1 deletion(-) diff --git a/csharp/ql/lib/semmle/code/csharp/Element.qll b/csharp/ql/lib/semmle/code/csharp/Element.qll index 38176acfd92..c44a092bb4f 100644 --- a/csharp/ql/lib/semmle/code/csharp/Element.qll +++ b/csharp/ql/lib/semmle/code/csharp/Element.qll @@ -118,7 +118,6 @@ class NamedElement extends Element, @named_element { * Unbound generic types, such as `IList`, are represented as * ``System.Collections.Generic.IList`1``. */ - cached deprecated final string getFullyQualifiedName() { exists(string qualifier, string name | this.hasFullyQualifiedName(qualifier, name) | if qualifier = "" then result = name else result = qualifier + "." + name From fc689efd1b1459e3da7771e46d364874d2544e97 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Mon, 8 Apr 2024 13:47:59 +0200 Subject: [PATCH 54/97] C#: Add debug version of the getFullyQualifiedName predicate. --- csharp/ql/lib/semmle/code/csharp/Element.qll | 27 ++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/csharp/ql/lib/semmle/code/csharp/Element.qll b/csharp/ql/lib/semmle/code/csharp/Element.qll index c44a092bb4f..f8cb018be68 100644 --- a/csharp/ql/lib/semmle/code/csharp/Element.qll +++ b/csharp/ql/lib/semmle/code/csharp/Element.qll @@ -124,6 +124,33 @@ class NamedElement extends Element, @named_element { ) } + /** + * INTERNAL: Do not use. + * + * This is intended for DEBUG ONLY. + * Constructing the fully qualified name for all elements in a large codebase + * puts severe stress on the string pool. + * + * Gets the fully qualified name of this element, for example the + * fully qualified name of `M` on line 3 is `N.C.M` in + * + * ```csharp + * namespace N { + * class C { + * void M(int i, string s) { } + * } + * } + * ``` + * + * Unbound generic types, such as `IList`, are represented as + * ``System.Collections.Generic.IList`1``. + */ + final string getFullyQualifiedNameDebug() { + exists(string qualifier, string name | this.hasFullyQualifiedName(qualifier, name) | + if qualifier = "" then result = name else result = qualifier + "." + name + ) + } + /** Holds if this element has the fully qualified name `qualifier`.`name`. */ cached predicate hasFullyQualifiedName(string qualifier, string name) { From 8cb6598f508a49848027fa1b8a5f583aef10f566 Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Mon, 8 Apr 2024 20:51:19 +0200 Subject: [PATCH 55/97] fixing that I put a type on the wrong thing in the alert-message --- java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql | 4 ++-- .../CWE-190/semmle/tests/InformationLoss.expected | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql b/java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql index e8893f3bcc7..7d97af12b71 100644 --- a/java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql +++ b/java/ql/src/Likely Bugs/Arithmetic/InformationLoss.ql @@ -45,5 +45,5 @@ where v = a.getDest() ) select a, - "Implicit cast of $@ to narrower destination type " + a.getDest().getType().getName() + ".", v, - "source type " + e.getType().getName() + "Implicit cast of source type " + e.getType().getName() + " to narrower destination type $@.", v, + a.getDest().getType().getName() diff --git a/java/ql/test/query-tests/security/CWE-190/semmle/tests/InformationLoss.expected b/java/ql/test/query-tests/security/CWE-190/semmle/tests/InformationLoss.expected index dbda11bff08..e317375c199 100644 --- a/java/ql/test/query-tests/security/CWE-190/semmle/tests/InformationLoss.expected +++ b/java/ql/test/query-tests/security/CWE-190/semmle/tests/InformationLoss.expected @@ -1,4 +1,4 @@ -| Test.java:68:5:68:25 | ...+=... | Implicit cast of $@ to narrower destination type int. | Test.java:64:4:64:13 | int i | source type long | -| Test.java:87:4:87:9 | ...+=... | Implicit cast of $@ to narrower destination type int. | Test.java:81:4:81:13 | int i | source type long | -| Test.java:289:5:289:30 | ...+=... | Implicit cast of $@ to narrower destination type int. | Test.java:285:4:285:27 | int[] arr | source type long | -| Test.java:293:7:293:44 | ...+=... | Implicit cast of $@ to narrower destination type int. | Test.java:293:7:293:24 | ...[...] | source type long | +| Test.java:68:5:68:25 | ...+=... | Implicit cast of source type long to narrower destination type $@. | Test.java:64:4:64:13 | int i | int | +| Test.java:87:4:87:9 | ...+=... | Implicit cast of source type long to narrower destination type $@. | Test.java:81:4:81:13 | int i | int | +| Test.java:289:5:289:30 | ...+=... | Implicit cast of source type long to narrower destination type $@. | Test.java:285:4:285:27 | int[] arr | int | +| Test.java:293:7:293:44 | ...+=... | Implicit cast of source type long to narrower destination type $@. | Test.java:293:7:293:24 | ...[...] | int | From c389611e5c5860637fcce06834c57e4d0c217d53 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Mon, 8 Apr 2024 17:07:48 +0100 Subject: [PATCH 56/97] C++: Add spurious dataflow test. --- .../dataflow-consistency.expected | 8 +++++++ .../dataflow-tests/test-source-sink.expected | 2 ++ .../dataflow/dataflow-tests/test.cpp | 24 ++++++++++++++++++- .../dataflow-tests/type-bugs.expected | 1 + 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/cpp/ql/test/library-tests/dataflow/dataflow-tests/dataflow-consistency.expected b/cpp/ql/test/library-tests/dataflow/dataflow-tests/dataflow-consistency.expected index fa6958d92ea..109f5ffebd1 100644 --- a/cpp/ql/test/library-tests/dataflow/dataflow-tests/dataflow-consistency.expected +++ b/cpp/ql/test/library-tests/dataflow/dataflow-tests/dataflow-consistency.expected @@ -33,6 +33,7 @@ argHasPostUpdate | test.cpp:67:29:67:35 | source1 | ArgumentNode is missing PostUpdateNode. | | test.cpp:813:19:813:35 | * ... | ArgumentNode is missing PostUpdateNode. | | test.cpp:848:23:848:25 | rpx | ArgumentNode is missing PostUpdateNode. | +| test.cpp:1057:19:1057:21 | * ... | ArgumentNode is missing PostUpdateNode. | postWithInFlow | BarrierGuard.cpp:49:6:49:6 | x [post update] | PostUpdateNode should not be the target of local flow. | | BarrierGuard.cpp:60:7:60:7 | x [post update] | PostUpdateNode should not be the target of local flow. | @@ -168,6 +169,13 @@ postWithInFlow | test.cpp:1045:9:1045:11 | ref arg buf | PostUpdateNode should not be the target of local flow. | | test.cpp:1051:5:1051:11 | content [post update] | PostUpdateNode should not be the target of local flow. | | test.cpp:1052:9:1052:9 | a [inner post update] | PostUpdateNode should not be the target of local flow. | +| test.cpp:1056:5:1056:7 | * ... [post update] | PostUpdateNode should not be the target of local flow. | +| test.cpp:1056:6:1056:7 | pp [inner post update] | PostUpdateNode should not be the target of local flow. | +| test.cpp:1062:53:1062:53 | p [inner post update] | PostUpdateNode should not be the target of local flow. | +| test.cpp:1072:3:1072:4 | * ... [post update] | PostUpdateNode should not be the target of local flow. | +| test.cpp:1072:4:1072:4 | p [inner post update] | PostUpdateNode should not be the target of local flow. | +| test.cpp:1073:3:1073:4 | * ... [post update] | PostUpdateNode should not be the target of local flow. | +| test.cpp:1073:4:1073:4 | p [inner post update] | PostUpdateNode should not be the target of local flow. | viableImplInCallContextTooLarge uniqueParameterNodeAtPosition uniqueParameterNodePosition diff --git a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test-source-sink.expected b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test-source-sink.expected index e03ee68b8a3..ada532793c4 100644 --- a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test-source-sink.expected +++ b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test-source-sink.expected @@ -310,6 +310,8 @@ irFlow | test.cpp:1021:18:1021:32 | *call to indirect_source | test.cpp:1027:19:1027:28 | *translated | | test.cpp:1021:18:1021:32 | *call to indirect_source | test.cpp:1031:19:1031:28 | *translated | | test.cpp:1045:14:1045:19 | call to source | test.cpp:1046:7:1046:10 | * ... | +| test.cpp:1061:15:1061:38 | *call to indirect_source | test.cpp:1057:19:1057:21 | ** ... | +| test.cpp:1072:8:1072:13 | call to source | test.cpp:1074:8:1074:9 | * ... | | true_upon_entry.cpp:9:11:9:16 | call to source | true_upon_entry.cpp:13:8:13:8 | x | | true_upon_entry.cpp:17:11:17:16 | call to source | true_upon_entry.cpp:21:8:21:8 | x | | true_upon_entry.cpp:27:9:27:14 | call to source | true_upon_entry.cpp:29:8:29:8 | x | diff --git a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp index b2bff6327c5..5e3b502468b 100644 --- a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp +++ b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp @@ -1050,4 +1050,26 @@ void flow_out_of_address_with_local_flow() { MyStruct a; a.content = nullptr; sink(&a); // $ SPURIOUS: ast -} \ No newline at end of file +} + +static void static_func_that_reassigns_pointer_before_sink(char** pp) { // $ ast-def=pp ir-def=*pp ir-def=**pp + *pp = ""; + indirect_sink(*pp); // $ SPURIOUS: ir +} + +void test_static_func_that_reassigns_pointer_before_sink() { + char* p = (char*)indirect_source(); + static_func_that_reassigns_pointer_before_sink(&p); +} + +void single_object_in_both_cases(bool b, int x, int y) { + int* p; + if(b) { + p = &x; + } else { + p = &y; + } + *p = source(); + *p = 0; + sink(*p); // $ SPURIOUS: ir +} diff --git a/cpp/ql/test/library-tests/dataflow/dataflow-tests/type-bugs.expected b/cpp/ql/test/library-tests/dataflow/dataflow-tests/type-bugs.expected index 6706d79e902..4d87c2da534 100644 --- a/cpp/ql/test/library-tests/dataflow/dataflow-tests/type-bugs.expected +++ b/cpp/ql/test/library-tests/dataflow/dataflow-tests/type-bugs.expected @@ -15,4 +15,5 @@ incorrectBaseType | test.cpp:848:23:848:25 | (reference dereference) | Expected 'Node.getType()' to be int, but it was int * | | test.cpp:854:10:854:36 | * ... | Expected 'Node.getType()' to be const int, but it was int | | test.cpp:867:10:867:30 | * ... | Expected 'Node.getType()' to be const int, but it was int | +| test.cpp:1062:52:1062:53 | *& ... | Expected 'Node.getType()' to be char, but it was char * | failures From b2002a981a6f6a9f2d69dfd38bfd9cc8c58f3eb0 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Mon, 8 Apr 2024 17:08:05 +0100 Subject: [PATCH 57/97] C++: Use the shared typeflow library to determine whether a pointer points to a buffer or an object. --- cpp/ql/lib/qlpack.yml | 1 + .../dataflow/internal/SsaInternalsCommon.qll | 7 +- .../cpp/ir/dataflow/internal/TypeFlow.qll | 259 ++++++++++++++++++ 3 files changed, 262 insertions(+), 5 deletions(-) create mode 100644 cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll diff --git a/cpp/ql/lib/qlpack.yml b/cpp/ql/lib/qlpack.yml index 8d15b6d142a..27529f55a29 100644 --- a/cpp/ql/lib/qlpack.yml +++ b/cpp/ql/lib/qlpack.yml @@ -9,6 +9,7 @@ dependencies: codeql/dataflow: ${workspace} codeql/rangeanalysis: ${workspace} codeql/ssa: ${workspace} + codeql/typeflow: ${workspace} codeql/tutorial: ${workspace} codeql/util: ${workspace} codeql/xml: ${workspace} diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternalsCommon.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternalsCommon.qll index 862070820f8..7e5b4de8122 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternalsCommon.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternalsCommon.qll @@ -6,6 +6,7 @@ private import DataFlowImplCommon as DataFlowImplCommon private import DataFlowUtil private import semmle.code.cpp.models.interfaces.PointerWrapper private import DataFlowPrivate +private import TypeFlow private import semmle.code.cpp.ir.ValueNumbering /** @@ -955,11 +956,7 @@ private module Cached { * Holds if the address computed by `operand` is guaranteed to write * to a specific address. */ - private predicate isCertainAddress(Operand operand) { - valueNumberOfOperand(operand).getAnInstruction() instanceof VariableAddressInstruction - or - operand.getType() instanceof Cpp::ReferenceType - } + private predicate isCertainAddress(Operand operand) { isPointerToSingleObject(operand.getDef()) } /** * Holds if `address` is a use of an SSA variable rooted at `base`, and the diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll new file mode 100644 index 00000000000..44f2e60daac --- /dev/null +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll @@ -0,0 +1,259 @@ +private import cpp +private import semmle.code.cpp.ir.IR +private import codeql.typeflow.TypeFlow + +private module Input implements TypeFlowInput { + private predicate hasExactSingleType(Instruction i) { + // The address of a variable is always a single object + i instanceof VariableAddressInstruction + or + // A reference always points to a always a single object + i.getResultLanguageType().hasUnspecifiedType(any(ReferenceType rt), false) + or + // `this` is never an array + i instanceof InitializeThisInstruction + or + // An allocation of a non-array object + exists(AllocationExpr alloc | alloc = i.getUnconvertedResultExpression() | + // i.e., `new int`; + alloc instanceof NewExpr + or + // i.e., `malloc(sizeof(int))` + exists(SizeofTypeOperator sizeOf | sizeOf = alloc.getSizeExpr() | + not sizeOf.getTypeOperand().getUnspecifiedType() instanceof ArrayType + ) + ) + } + + private predicate hasExactBufferType(Instruction i) { + // Anything with an array type is a buffer + i.getResultLanguageType().hasUnspecifiedType(any(ArrayType at), false) + or + not hasExactSingleType(i) and + i.getUnconvertedResultExpression() instanceof AllocationExpr + } + + private newtype TTypeFlowNode = + TInstructionNode(Instruction i) or + TFunctionNode(IRFunction func) + + abstract class TypeFlowNode extends TTypeFlowNode { + /** Gets a textual representation of this node. */ + abstract string toString(); + + /** + * Gets the type of this node. This type may not be the most precise + * possible type, but will be used as a starting point of the analysis. + */ + abstract Type getType(); + + /** Gets the location of this node. */ + abstract Location getLocation(); + + /** Gets the underlying `Instruction` of this node, if any. */ + Instruction asInstruction() { none() } + + /** Gets the underlying `IRFunction` of this node, if any. */ + IRFunction asFunction() { none() } + + /** Holds if the value of this node is always null. */ + abstract predicate isNullValue(); + } + + private class InstructionNode extends TypeFlowNode, TInstructionNode { + Instruction instr; + + InstructionNode() { this = TInstructionNode(instr) } + + override string toString() { result = instr.toString() } + + override Type getType() { + if hasExactSingleType(instr) then result.isSingle() else result.isBuffer() + } + + override Location getLocation() { result = instr.getLocation() } + + override Instruction asInstruction() { result = instr } + + override predicate isNullValue() { + instr.(ConstantInstruction).getValue() = "0" and + instr.getResultIRType() instanceof IRAddressType + } + } + + /** Gets the `TypeFlowNode` corresponding to `i`. */ + additional InstructionNode instructionNode(Instruction i) { result.asInstruction() = i } + + private class FunctionNode extends TypeFlowNode, TFunctionNode { + IRFunction func; + + FunctionNode() { this = TFunctionNode(func) } + + override string toString() { result = func.toString() } + + Instruction getReturnValueInstruction() { + result = func.getReturnInstruction().(ReturnValueInstruction).getReturnValue() + } + + override Type getType() { result = instructionNode(this.getReturnValueInstruction()).getType() } + + override Location getLocation() { result = func.getLocation() } + + override IRFunction asFunction() { result = func } + + override predicate isNullValue() { + instructionNode(this.getReturnValueInstruction()).isNullValue() + } + } + + /** + * Gets an ultimiate definition of `phi`. That is, an input to `phi` that is + * not itself a `PhiInstruction`. + */ + private Instruction getAnUltimateLocalDefinition(PhiInstruction phi) { + result = phi.getAnInput*() and not result instanceof PhiInstruction + } + + /** + * Holds if this function is private (i.e., cannot be accessed outside its + * compilation unit). This means we can use a closed-world assumption about + * calls to this function. + */ + private predicate isPrivate(Function func) { + func.isStatic() + or + func.getNamespace().getParentNamespace*().isInline() + or + func.(MemberFunction).isPrivate() + } + + /** + * Holds if `arg` is an argument for the parameter `p` in a private callable. + */ + pragma[nomagic] + private predicate privateParamArg(InitializeParameterInstruction p, Instruction arg) { + exists(CallInstruction call, int i, Function func | + call.getArgument(pragma[only_bind_into](i)) = arg and + func = call.getStaticCallTarget() and + func.getParameter(pragma[only_bind_into](i)) = p.getParameter() and + isPrivate(func) + ) + } + + predicate joinStep(TypeFlowNode n1, TypeFlowNode n2) { + // instruction -> phi + getAnUltimateLocalDefinition(n2.asInstruction()) = n1.asInstruction() + or + // return value -> function + n2.(FunctionNode).getReturnValueInstruction() = n1.asInstruction() + or + // function -> call + exists(Function func | func = n1.asFunction().getFunction() | + not func.isVirtual() and + n2.asInstruction().(CallInstruction).getStaticCallTarget() = func + ) + or + // Argument -> parameter where the parameter's enclosing function + // is "private". + exists(Instruction arg, Instruction p | + privateParamArg(p, arg) and + n1.asInstruction() = arg and + n2.asInstruction() = p + ) + } + + /** + * Holds if knowing whether `i1` points to a single object or buffer implies + * knowing whether `i2` points to a single object or buffer. + */ + private predicate instructionStep(Instruction i1, Instruction i2) { + i2.(CopyInstruction).getSourceValue() = i1 + or + i2.(CopyValueInstruction).getSourceValue() = i1 + or + i2.(ConvertInstruction).getUnary() = i1 + or + i2.(CheckedConvertOrNullInstruction).getUnary() = i1 + or + i2.(InheritanceConversionInstruction).getUnary() = i1 + or + i2.(PointerArithmeticInstruction).getLeft() = i1 + } + + predicate step(TypeFlowNode n1, TypeFlowNode n2) { + instructionStep(n1.asInstruction(), n2.asInstruction()) + } + + predicate isNullValue(TypeFlowNode n) { n.isNullValue() } + + private newtype TType = + TSingle() or + TBuffer() + + class Type extends TType { + string toString() { + this.isSingle() and + result = "Single" + or + this.isBuffer() and + result = "Buffer" + } + + /** Holds if this type is the type that represents a single object. */ + predicate isSingle() { this = TSingle() } + + /** Holds if this type is the type that represents a buffer. */ + predicate isBuffer() { this = TBuffer() } + + /** + * Gets a super type of this type, if any. + * + * The type relation is `Single <: Buffer`. + */ + Type getASupertype() { + this.isSingle() and + result.isBuffer() + } + } + + predicate exactTypeBase(TypeFlowNode n, Type t) { + exists(Instruction instr | instr = n.asInstruction() | + hasExactSingleType(instr) and t.isSingle() + or + hasExactBufferType(instr) and t.isBuffer() + ) + } + + pragma[nomagic] + private predicate upcastCand(TypeFlowNode n, Type t1, Type t2) { + exists(TypeFlowNode next | + step(n, next) + or + joinStep(n, next) + | + n.getType() = t1 and + next.getType() = t2 and + t1 != t2 + ) + } + + private predicate upcast(TypeFlowNode n, Type t1) { + exists(Type t2 | upcastCand(n, t1, t2) | + // No need for transitive closure since the subtyping relation is just `Single <: Buffer` + t1.getASupertype() = t2 + ) + } + + predicate typeFlowBaseCand(TypeFlowNode n, Type t) { upcast(n, t) } +} + +private module TypeFlow = Make; + +/** + * Holds if `i` is an instruction that computes an address that points to a + * single object (as opposed to pointing into a buffer). + */ +pragma[nomagic] +predicate isPointerToSingleObject(Instruction i) { + TypeFlow::bestTypeFlow(Input::instructionNode(i), any(Input::Type t | t.isSingle()), _) +} From 386580fc94750a0036d0b27e21ff05d86dacdb2c Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Mon, 8 Apr 2024 17:10:18 +0100 Subject: [PATCH 58/97] C++: Accept test changes. --- .../dataflow/dataflow-tests/test-source-sink.expected | 2 -- cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test-source-sink.expected b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test-source-sink.expected index ada532793c4..e03ee68b8a3 100644 --- a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test-source-sink.expected +++ b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test-source-sink.expected @@ -310,8 +310,6 @@ irFlow | test.cpp:1021:18:1021:32 | *call to indirect_source | test.cpp:1027:19:1027:28 | *translated | | test.cpp:1021:18:1021:32 | *call to indirect_source | test.cpp:1031:19:1031:28 | *translated | | test.cpp:1045:14:1045:19 | call to source | test.cpp:1046:7:1046:10 | * ... | -| test.cpp:1061:15:1061:38 | *call to indirect_source | test.cpp:1057:19:1057:21 | ** ... | -| test.cpp:1072:8:1072:13 | call to source | test.cpp:1074:8:1074:9 | * ... | | true_upon_entry.cpp:9:11:9:16 | call to source | true_upon_entry.cpp:13:8:13:8 | x | | true_upon_entry.cpp:17:11:17:16 | call to source | true_upon_entry.cpp:21:8:21:8 | x | | true_upon_entry.cpp:27:9:27:14 | call to source | true_upon_entry.cpp:29:8:29:8 | x | diff --git a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp index 5e3b502468b..af9e18034ed 100644 --- a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp +++ b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp @@ -1054,7 +1054,7 @@ void flow_out_of_address_with_local_flow() { static void static_func_that_reassigns_pointer_before_sink(char** pp) { // $ ast-def=pp ir-def=*pp ir-def=**pp *pp = ""; - indirect_sink(*pp); // $ SPURIOUS: ir + indirect_sink(*pp); // clean } void test_static_func_that_reassigns_pointer_before_sink() { @@ -1071,5 +1071,5 @@ void single_object_in_both_cases(bool b, int x, int y) { } *p = source(); *p = 0; - sink(*p); // $ SPURIOUS: ir + sink(*p); // clean } From e3d676f91bdac5130e5bcfc83c90b85c3715749f Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 9 Apr 2024 09:37:21 +0200 Subject: [PATCH 59/97] CI: apply tentative `setup-swift` fix --- swift/actions/run-integration-tests/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift/actions/run-integration-tests/action.yml b/swift/actions/run-integration-tests/action.yml index 2c6df4e2b51..fe5a20b02bd 100644 --- a/swift/actions/run-integration-tests/action.yml +++ b/swift/actions/run-integration-tests/action.yml @@ -7,7 +7,7 @@ runs: - uses: actions/setup-python@v4 with: python-version-file: 'swift/.python-version' - - uses: swift-actions/setup-swift@65540b95f51493d65f5e59e97dcef9629ddf11bf + - uses: redsun82/setup-swift@b2b6f77ab14f6a9b136b520dc53ec8eca27d2b99 with: swift-version: "5.8" - uses: ./.github/actions/fetch-codeql From 80995ec1d76b645097159a140e5d44148bb77b25 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Tue, 9 Apr 2024 09:51:45 +0200 Subject: [PATCH 60/97] Improve comments on environment variable names --- .../EnvironmentVariableNames.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs index ddec97d3dde..9141dc0bf74 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs @@ -23,12 +23,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching public const string CheckNugetFeedResponsiveness = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK"; /// - /// Specifies the NuGet feeds to exclude from the responsiveness check. + /// Specifies the NuGet feeds to exclude from the responsiveness check. The value is a space-separated list of feed URLs. /// public const string ExcludedNugetFeedsFromResponsivenessCheck = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_EXCLUDED"; /// - /// Specifies the timeout for the initial check of NuGet feeds responsiveness. + /// Specifies the timeout (as an integer) in milliseconds for the initial check of NuGet feeds responsiveness. The value is then doubled for each subsequent check. /// public const string NugetFeedResponsivenessInitialTimeout = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT"; From e6984aa865e23a48215c7280258af247892db8b6 Mon Sep 17 00:00:00 2001 From: Tom Hvitved Date: Tue, 9 Apr 2024 10:10:25 +0200 Subject: [PATCH 61/97] Ruby: Remove two redundant `allowImplicitRead` predicates --- .../codeql/ruby/security/UnsafeCodeConstructionQuery.qll | 6 ------ .../ruby/security/UnsafeShellCommandConstructionQuery.qll | 6 ------ 2 files changed, 12 deletions(-) diff --git a/ruby/ql/lib/codeql/ruby/security/UnsafeCodeConstructionQuery.qll b/ruby/ql/lib/codeql/ruby/security/UnsafeCodeConstructionQuery.qll index 4cf57f36071..32cc9a4f821 100644 --- a/ruby/ql/lib/codeql/ruby/security/UnsafeCodeConstructionQuery.qll +++ b/ruby/ql/lib/codeql/ruby/security/UnsafeCodeConstructionQuery.qll @@ -46,12 +46,6 @@ private module UnsafeCodeConstructionConfig implements DataFlow::ConfigSig { // override to require the path doesn't have unmatched return steps DataFlow::FlowFeature getAFeature() { result instanceof DataFlow::FeatureHasSourceCallContext } - - predicate allowImplicitRead(DataFlow::Node node, DataFlow::ContentSet set) { - // allow implicit reads of array elements - isSink(node) and - set.isElementOfTypeOrUnknown("int") - } } /** diff --git a/ruby/ql/lib/codeql/ruby/security/UnsafeShellCommandConstructionQuery.qll b/ruby/ql/lib/codeql/ruby/security/UnsafeShellCommandConstructionQuery.qll index 7576702a2d4..b4e0b8b6bb5 100644 --- a/ruby/ql/lib/codeql/ruby/security/UnsafeShellCommandConstructionQuery.qll +++ b/ruby/ql/lib/codeql/ruby/security/UnsafeShellCommandConstructionQuery.qll @@ -49,12 +49,6 @@ private module UnsafeShellCommandConstructionConfig implements DataFlow::ConfigS // override to require the path doesn't have unmatched return steps DataFlow::FlowFeature getAFeature() { result instanceof DataFlow::FeatureHasSourceCallContext } - - predicate allowImplicitRead(DataFlow::Node node, DataFlow::ContentSet set) { - // allow implicit reads of array elements - isSink(node) and - set.isElementOfTypeOrUnknown("int") - } } /** From 10d96ee02f6fa0e84693135ffb830acc35d2cb8e Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Tue, 9 Apr 2024 10:31:48 +0200 Subject: [PATCH 62/97] C#: Address review comments. --- .../diag_recursive_generics/Types.ql | 7 +++---- csharp/ql/lib/semmle/code/csharp/Element.qll | 2 ++ .../ql/lib/semmle/code/csharp/dataflow/SSA.qll | 8 +------- .../attributes/AttributeElements.ql | 8 +++----- .../library-tests/constructors/Destructors1.ql | 3 +-- .../test/library-tests/csharp11/fileScoped.ql | 6 +----- .../ql/test/library-tests/csharp11/nativeInt.ql | 6 ++---- .../library-tests/csharp9/covariantReturn.ql | 13 ++++--------- csharp/ql/test/library-tests/csharp9/foreach.ql | 17 ++++++----------- csharp/ql/test/library-tests/csharp9/record.ql | 10 +--------- .../ql/test/library-tests/csharp9/withExpr.ql | 14 +++----------- csharp/ql/test/library-tests/enums/Enums3.ql | 8 +++----- .../ql/test/library-tests/generics/Generics.ql | 12 ++++-------- 13 files changed, 34 insertions(+), 80 deletions(-) diff --git a/csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql b/csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql index fff011dcbd5..de95e0fcbe7 100644 --- a/csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql +++ b/csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql @@ -1,6 +1,5 @@ import csharp -import semmle.code.csharp.commons.QualifiedName -from Class c, string qualifier, string name -where c.fromSource() and c.getBaseClass().hasFullyQualifiedName(qualifier, name) -select c, getQualifiedName(qualifier, name) +from Class c +where c.fromSource() +select c, c.getBaseClass().getFullyQualifiedNameDebug() diff --git a/csharp/ql/lib/semmle/code/csharp/Element.qll b/csharp/ql/lib/semmle/code/csharp/Element.qll index f8cb018be68..a48241a1408 100644 --- a/csharp/ql/lib/semmle/code/csharp/Element.qll +++ b/csharp/ql/lib/semmle/code/csharp/Element.qll @@ -145,6 +145,8 @@ class NamedElement extends Element, @named_element { * Unbound generic types, such as `IList`, are represented as * ``System.Collections.Generic.IList`1``. */ + bindingset[this] + pragma[inline_late] final string getFullyQualifiedNameDebug() { exists(string qualifier, string name | this.hasFullyQualifiedName(qualifier, name) | if qualifier = "" then result = name else result = qualifier + "." + name diff --git a/csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll b/csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll index 9cd33ea260f..0d79eafdf5c 100644 --- a/csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll +++ b/csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll @@ -3,7 +3,6 @@ */ import csharp -private import semmle.code.csharp.commons.QualifiedName /** * Provides classes for working with static single assignment (SSA) form. @@ -121,12 +120,7 @@ module Ssa { result = prefix + "." + this.getAssignable() | if f.(Modifiable).isStatic() - then - exists(string qualifier, string name | - f.getDeclaringType().hasFullyQualifiedName(qualifier, name) - | - prefix = getQualifiedName(qualifier, name) - ) + then prefix = f.getDeclaringType().getName() else prefix = "this" ) } diff --git a/csharp/ql/test/library-tests/attributes/AttributeElements.ql b/csharp/ql/test/library-tests/attributes/AttributeElements.ql index 17ce2ea3d93..679d7567ea5 100644 --- a/csharp/ql/test/library-tests/attributes/AttributeElements.ql +++ b/csharp/ql/test/library-tests/attributes/AttributeElements.ql @@ -1,9 +1,7 @@ import csharp -import semmle.code.csharp.commons.QualifiedName -from Attributable element, Attribute attribute, string qualifier, string name +from Attributable element, Attribute attribute where attribute = element.getAnAttribute() and - (attribute.fromSource() or element.(Assembly).getName() in ["attributes", "Assembly1"]) and - attribute.getType().hasFullyQualifiedName(qualifier, name) -select element, attribute, getQualifiedName(qualifier, name) + (attribute.fromSource() or element.(Assembly).getName() in ["attributes", "Assembly1"]) +select element, attribute, attribute.getType().getFullyQualifiedNameDebug() diff --git a/csharp/ql/test/library-tests/constructors/Destructors1.ql b/csharp/ql/test/library-tests/constructors/Destructors1.ql index 673a0e941f0..792d50da7bb 100644 --- a/csharp/ql/test/library-tests/constructors/Destructors1.ql +++ b/csharp/ql/test/library-tests/constructors/Destructors1.ql @@ -3,11 +3,10 @@ */ import csharp -import semmle.code.csharp.commons.QualifiedName from Destructor c, string qualifier, string name where c.getDeclaringType().hasFullyQualifiedName(qualifier, name) and qualifier = "Constructors" and name = "Class" -select c, getQualifiedName(qualifier, name) +select c, c.getDeclaringType().getFullyQualifiedNameDebug() diff --git a/csharp/ql/test/library-tests/csharp11/fileScoped.ql b/csharp/ql/test/library-tests/csharp11/fileScoped.ql index 3003fc801bf..33697255896 100644 --- a/csharp/ql/test/library-tests/csharp11/fileScoped.ql +++ b/csharp/ql/test/library-tests/csharp11/fileScoped.ql @@ -1,5 +1,4 @@ import csharp -private import semmle.code.csharp.commons.QualifiedName private predicate isInteresting(Type t) { ( @@ -20,10 +19,7 @@ query predicate typemodifiers(Type t, string modifier) { query predicate qualifiedtypes(Type t, string qualifiedName) { isInteresting(t) and - exists(string qualifier, string name | - t.hasFullyQualifiedName(qualifier, name) and - qualifiedName = getQualifiedName(qualifier, name) - ) + qualifiedName = t.getFullyQualifiedNameDebug() } query predicate filetypes(Type t) { diff --git a/csharp/ql/test/library-tests/csharp11/nativeInt.ql b/csharp/ql/test/library-tests/csharp11/nativeInt.ql index 80d7974de56..adbc062baf6 100644 --- a/csharp/ql/test/library-tests/csharp11/nativeInt.ql +++ b/csharp/ql/test/library-tests/csharp11/nativeInt.ql @@ -1,12 +1,10 @@ import csharp -import semmle.code.csharp.commons.QualifiedName -from LocalVariable v1, LocalVariable v2, Type t, string qualifier, string name +from LocalVariable v1, LocalVariable v2, Type t where v1.getFile().getStem() = "NativeInt" and v2.getFile().getStem() = "NativeInt" and t = v1.getType() and t = v2.getType() and - t.hasFullyQualifiedName(qualifier, name) and v1 != v2 -select v1, v2, getQualifiedName(qualifier, name) +select v1, v2, t.getFullyQualifiedNameDebug() diff --git a/csharp/ql/test/library-tests/csharp9/covariantReturn.ql b/csharp/ql/test/library-tests/csharp9/covariantReturn.ql index 6227ed18d6d..b4bab047322 100644 --- a/csharp/ql/test/library-tests/csharp9/covariantReturn.ql +++ b/csharp/ql/test/library-tests/csharp9/covariantReturn.ql @@ -1,13 +1,8 @@ import csharp -import semmle.code.csharp.commons.QualifiedName -from - Method m, Method overrider, string mnamespace, string mtype, string mname, string onamespace, - string otype, string oname +from Method m, Method overrider where m.getAnOverrider() = overrider and - m.getFile().getStem() = "CovariantReturn" and - m.hasFullyQualifiedName(mnamespace, mtype, mname) and - overrider.hasFullyQualifiedName(onamespace, otype, oname) -select getQualifiedName(mnamespace, mtype, mname), m.getReturnType().toString(), - getQualifiedName(onamespace, otype, oname), overrider.getReturnType().toString() + m.getFile().getStem() = "CovariantReturn" +select m.getFullyQualifiedNameDebug(), m.getReturnType().toString(), + overrider.getFullyQualifiedNameDebug(), overrider.getReturnType().toString() diff --git a/csharp/ql/test/library-tests/csharp9/foreach.ql b/csharp/ql/test/library-tests/csharp9/foreach.ql index cf6fcf2514a..343ecc556ab 100644 --- a/csharp/ql/test/library-tests/csharp9/foreach.ql +++ b/csharp/ql/test/library-tests/csharp9/foreach.ql @@ -1,5 +1,4 @@ import csharp -import semmle.code.csharp.commons.QualifiedName private string getLocation(Member m) { if m.fromSource() then result = m.getALocation().(SourceLocation).toString() else result = "-" @@ -9,13 +8,9 @@ private string getIsAsync(ForeachStmt f) { if f.isAsync() then result = "async" else result = "sync" } -from - ForeachStmt f, string qualifier1, string type1, string qualifier2, string type2, - string qualifier3, string type3 -where - f.getGetEnumerator().getDeclaringType().hasFullyQualifiedName(qualifier1, type1) and - f.getCurrent().getDeclaringType().hasFullyQualifiedName(qualifier2, type2) and - f.getMoveNext().getDeclaringType().hasFullyQualifiedName(qualifier3, type3) -select f, f.getElementType().toString(), getIsAsync(f), getQualifiedName(qualifier1, type1), - getLocation(f.getGetEnumerator()), getQualifiedName(qualifier2, type2), - getLocation(f.getCurrent()), getQualifiedName(qualifier3, type3), getLocation(f.getMoveNext()) +from ForeachStmt f +select f, f.getElementType().toString(), getIsAsync(f), + f.getGetEnumerator().getDeclaringType().getFullyQualifiedNameDebug(), + getLocation(f.getGetEnumerator()), f.getCurrent().getDeclaringType().getFullyQualifiedNameDebug(), + getLocation(f.getCurrent()), f.getMoveNext().getDeclaringType().getFullyQualifiedNameDebug(), + getLocation(f.getMoveNext()) diff --git a/csharp/ql/test/library-tests/csharp9/record.ql b/csharp/ql/test/library-tests/csharp9/record.ql index a2a9a9c0786..58cf579bac6 100644 --- a/csharp/ql/test/library-tests/csharp9/record.ql +++ b/csharp/ql/test/library-tests/csharp9/record.ql @@ -7,18 +7,10 @@ query predicate records(RecordClass t, string i, RecordCloneMethod clone) { t.fromSource() } -private string getMemberName(Member m) { - exists(string qualifier, string name | - m.getDeclaringType().hasFullyQualifiedName(qualifier, name) - | - result = getQualifiedName(qualifier, name) + "." + m.toStringWithTypes() - ) -} - query predicate members(RecordClass t, string ms, string l) { t.fromSource() and exists(Member m | t.hasMember(m) | - ms = getMemberName(m) and + ms = getFullyQualifiedNameWithTypes(m) and if m.fromSource() then l = m.getLocation().toString() else l = "no location" ) } diff --git a/csharp/ql/test/library-tests/csharp9/withExpr.ql b/csharp/ql/test/library-tests/csharp9/withExpr.ql index 6683d7c54f6..564cbe529aa 100644 --- a/csharp/ql/test/library-tests/csharp9/withExpr.ql +++ b/csharp/ql/test/library-tests/csharp9/withExpr.ql @@ -1,19 +1,11 @@ import csharp import semmle.code.csharp.commons.QualifiedName -private string getSignature(Method m) { - exists(string qualifier, string name | - m.getDeclaringType().hasFullyQualifiedName(qualifier, name) - | - result = getQualifiedName(qualifier, name) + "." + m.toStringWithTypes() - ) -} - query predicate withExpr(WithExpr with, string type, Expr expr, ObjectInitializer init, string clone) { type = with.getType().toStringWithTypes() and expr = with.getExpr() and init = with.getInitializer() and - clone = getSignature(with.getCloneMethod()) + clone = getFullyQualifiedNameWithTypes(with.getCloneMethod()) } query predicate withTarget(WithExpr with, RecordCloneMethod clone, Constructor ctor) { @@ -25,7 +17,7 @@ query predicate cloneOverrides(string b, string o) { exists(RecordCloneMethod base, RecordCloneMethod overrider | base.getDeclaringType().fromSource() and base.getAnOverrider() = overrider and - b = getSignature(base) and - o = getSignature(overrider) + b = getFullyQualifiedNameWithTypes(base) and + o = getFullyQualifiedNameWithTypes(overrider) ) } diff --git a/csharp/ql/test/library-tests/enums/Enums3.ql b/csharp/ql/test/library-tests/enums/Enums3.ql index 5cfbdc56193..b01ffb97e18 100644 --- a/csharp/ql/test/library-tests/enums/Enums3.ql +++ b/csharp/ql/test/library-tests/enums/Enums3.ql @@ -3,13 +3,11 @@ */ import csharp -import semmle.code.csharp.commons.QualifiedName -from EnumConstant c, string namespace, string name +from EnumConstant c where c.getName() = "Green" and c.getDeclaringType().hasFullyQualifiedName("Enums", "LongColor") and c.getType() = c.getDeclaringType() and - c.getValue() = "1" and - c.getDeclaringType().getBaseClass().hasFullyQualifiedName(namespace, name) -select c, getQualifiedName(namespace, name) + c.getValue() = "1" +select c, c.getDeclaringType().getBaseClass().getFullyQualifiedNameDebug() diff --git a/csharp/ql/test/library-tests/generics/Generics.ql b/csharp/ql/test/library-tests/generics/Generics.ql index 71e21a5815e..e75f1fdc908 100644 --- a/csharp/ql/test/library-tests/generics/Generics.ql +++ b/csharp/ql/test/library-tests/generics/Generics.ql @@ -268,16 +268,12 @@ query predicate test33(ConstructedMethod cm, string s1, string s2) { query predicate test34(UnboundGeneric ug, string s1, string s2) { ug.fromSource() and - exists(string qualifier, string name | - ug.hasFullyQualifiedName(qualifier, name) and s1 = getQualifiedName(qualifier, name) - ) and - getFullyQualifiedNameWithTypes(ug) = s2 + s1 = ug.getFullyQualifiedNameDebug() and + s2 = getFullyQualifiedNameWithTypes(ug) } query predicate test35(UnboundGenericMethod gm, string s1, string s2) { gm.fromSource() and - exists(string namespace, string type, string name | - gm.hasFullyQualifiedName(namespace, type, name) and s1 = getQualifiedName(namespace, type, name) - ) and - getFullyQualifiedNameWithTypes(gm) = s2 + s1 = gm.getFullyQualifiedNameDebug() and + s2 = getFullyQualifiedNameWithTypes(gm) } From 8c2455fc113f3d0ee91b5d264c96782e06a6c274 Mon Sep 17 00:00:00 2001 From: Taus Date: Tue, 9 Apr 2024 10:49:30 +0000 Subject: [PATCH 63/97] Python: Disable failing integration tests These failures were likely caused by https://github.com/github/codeql/pull/16127 My guess is that they can probably be deleted altogether, but as the failures are blocking other development, I have opted to simply disable them for the time being. --- .../force-enable-library-extraction/{test.sh => disabled-test.sh} | 0 .../ignore-venv/{test.sh => disabled-test.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename python/extractor/cli-integration-test/force-enable-library-extraction/{test.sh => disabled-test.sh} (100%) rename python/extractor/cli-integration-test/ignore-venv/{test.sh => disabled-test.sh} (100%) diff --git a/python/extractor/cli-integration-test/force-enable-library-extraction/test.sh b/python/extractor/cli-integration-test/force-enable-library-extraction/disabled-test.sh similarity index 100% rename from python/extractor/cli-integration-test/force-enable-library-extraction/test.sh rename to python/extractor/cli-integration-test/force-enable-library-extraction/disabled-test.sh diff --git a/python/extractor/cli-integration-test/ignore-venv/test.sh b/python/extractor/cli-integration-test/ignore-venv/disabled-test.sh similarity index 100% rename from python/extractor/cli-integration-test/ignore-venv/test.sh rename to python/extractor/cli-integration-test/ignore-venv/disabled-test.sh From bb4952f5575e71c19473982a0287db8773b4fd5a Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 9 Apr 2024 14:00:25 +0200 Subject: [PATCH 64/97] Revert "Python: Disable failing integration tests" This reverts commit 8c2455fc113f3d0ee91b5d264c96782e06a6c274. --- .../force-enable-library-extraction/{disabled-test.sh => test.sh} | 0 .../ignore-venv/{disabled-test.sh => test.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename python/extractor/cli-integration-test/force-enable-library-extraction/{disabled-test.sh => test.sh} (100%) rename python/extractor/cli-integration-test/ignore-venv/{disabled-test.sh => test.sh} (100%) diff --git a/python/extractor/cli-integration-test/force-enable-library-extraction/disabled-test.sh b/python/extractor/cli-integration-test/force-enable-library-extraction/test.sh similarity index 100% rename from python/extractor/cli-integration-test/force-enable-library-extraction/disabled-test.sh rename to python/extractor/cli-integration-test/force-enable-library-extraction/test.sh diff --git a/python/extractor/cli-integration-test/ignore-venv/disabled-test.sh b/python/extractor/cli-integration-test/ignore-venv/test.sh similarity index 100% rename from python/extractor/cli-integration-test/ignore-venv/disabled-test.sh rename to python/extractor/cli-integration-test/ignore-venv/test.sh From a0d6324f685ccfbada6d682379101f6822dfef44 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 9 Apr 2024 14:01:10 +0200 Subject: [PATCH 65/97] Python: Fix ignore-venv integration test Now that we no longer support the fallback option (https://github.com/github/codeql/pull/16127) --- .../cli-integration-test/ignore-venv/test.sh | 32 +------------------ 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/python/extractor/cli-integration-test/ignore-venv/test.sh b/python/extractor/cli-integration-test/ignore-venv/test.sh index e1368008891..8735203ec32 100755 --- a/python/extractor/cli-integration-test/ignore-venv/test.sh +++ b/python/extractor/cli-integration-test/ignore-venv/test.sh @@ -17,63 +17,33 @@ mkdir dbs # set up venvs cd repo_dir +# make venv with some package in it (so we show that our ignore logic is correct) python3 -m venv venv venv/bin/pip install flask -python3 -m venv venv2 - cd "$SCRIPTDIR" -# In 2.16.0 we stop extracting libraries by default, so to test this functionality we -# need to force enable it. Once we release 2.17.0 and turn off library extraction for -# good, we can remove the part of this test ensuring that dependencies in an active -# venv are still extracted (since that will no longer be the case). -export CODEQL_EXTRACTOR_PYTHON_FORCE_ENABLE_LIBRARY_EXTRACTION_UNTIL_2_17_0=1 - -# Create DBs with venv2 active (that does not have flask installed) -source repo_dir/venv2/bin/activate - export CODEQL_EXTRACTOR_PYTHON_DISABLE_AUTOMATIC_VENV_EXCLUDE= $CODEQL database create dbs/normal --language python --source-root repo_dir/ export CODEQL_EXTRACTOR_PYTHON_DISABLE_AUTOMATIC_VENV_EXCLUDE=1 $CODEQL database create dbs/no-venv-ignore --language python --source-root repo_dir/ -# Create DB with venv active that has flask installed. We want to ensure that we're -# still able to resolve imports to flask, but don't want to extract EVERYTHING from -# within the venv. Important note is that the test-file in the repo_dir actually imports -# flask :D -source repo_dir/venv/bin/activate -export CODEQL_EXTRACTOR_PYTHON_DISABLE_AUTOMATIC_VENV_EXCLUDE= -$CODEQL database create dbs/normal-with-flask-venv --language python --source-root repo_dir/ - # --- set +x EXTRACTED_NORMAL=$(unzip -l dbs/normal/src.zip | wc -l) EXTRACTED_NO_VENV_IGNORE=$(unzip -l dbs/no-venv-ignore/src.zip | wc -l) -EXTRACTED_ACTIVE_FLASK=$(unzip -l dbs/normal-with-flask-venv/src.zip | wc -l) exitcode=0 echo "EXTRACTED_NORMAL=$EXTRACTED_NORMAL" echo "EXTRACTED_NO_VENV_IGNORE=$EXTRACTED_NO_VENV_IGNORE" -echo "EXTRACTED_ACTIVE_FLASK=$EXTRACTED_ACTIVE_FLASK" if [[ ! $EXTRACTED_NORMAL -lt $EXTRACTED_NO_VENV_IGNORE ]]; then echo "ERROR: EXTRACTED_NORMAL not smaller EXTRACTED_NO_VENV_IGNORE" exitcode=1 fi -if [[ ! $EXTRACTED_NORMAL -lt $EXTRACTED_ACTIVE_FLASK ]]; then - echo "ERROR: EXTRACTED_NORMAL not smaller EXTRACTED_ACTIVE_FLASK" - exitcode=1 -fi - -if [[ ! $EXTRACTED_ACTIVE_FLASK -lt $EXTRACTED_NO_VENV_IGNORE ]]; then - echo "ERROR: EXTRACTED_ACTIVE_FLASK not smaller EXTRACTED_NO_VENV_IGNORE" - exitcode=1 -fi - exit $exitcode From e9e7ccddcea1f00326522457c5a5d816564d55df Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 9 Apr 2024 14:02:34 +0200 Subject: [PATCH 66/97] Python: delete `force-enable-library-extraction` integration test --- .../repo_dir/foo.py | 3 -- .../force-enable-library-extraction/test.sh | 41 ------------------- 2 files changed, 44 deletions(-) delete mode 100644 python/extractor/cli-integration-test/force-enable-library-extraction/repo_dir/foo.py delete mode 100755 python/extractor/cli-integration-test/force-enable-library-extraction/test.sh diff --git a/python/extractor/cli-integration-test/force-enable-library-extraction/repo_dir/foo.py b/python/extractor/cli-integration-test/force-enable-library-extraction/repo_dir/foo.py deleted file mode 100644 index cf0cd77a108..00000000000 --- a/python/extractor/cli-integration-test/force-enable-library-extraction/repo_dir/foo.py +++ /dev/null @@ -1,3 +0,0 @@ -import pip - -print(42) diff --git a/python/extractor/cli-integration-test/force-enable-library-extraction/test.sh b/python/extractor/cli-integration-test/force-enable-library-extraction/test.sh deleted file mode 100755 index 9d74cfaca4b..00000000000 --- a/python/extractor/cli-integration-test/force-enable-library-extraction/test.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -set -Eeuo pipefail # see https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ - -set -x - -CODEQL=${CODEQL:-codeql} - -SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$SCRIPTDIR" - -# start on clean slate -rm -rf dbs -mkdir dbs - -cd "$SCRIPTDIR" - -export CODEQL_EXTRACTOR_PYTHON_FORCE_ENABLE_LIBRARY_EXTRACTION_UNTIL_2_17_0= -$CODEQL database create dbs/normal --language python --source-root repo_dir/ - -export CODEQL_EXTRACTOR_PYTHON_FORCE_ENABLE_LIBRARY_EXTRACTION_UNTIL_2_17_0=1 -$CODEQL database create dbs/with-lib-extraction --language python --source-root repo_dir/ - -# --- - -set +x - -EXTRACTED_NORMAL=$(unzip -l dbs/normal/src.zip | wc -l) -EXTRACTED_WITH_LIB_EXTRACTION=$(unzip -l dbs/with-lib-extraction/src.zip | wc -l) - -exitcode=0 - -echo "EXTRACTED_NORMAL=$EXTRACTED_NORMAL" -echo "EXTRACTED_WITH_LIB_EXTRACTION=$EXTRACTED_WITH_LIB_EXTRACTION" - -if [[ ! $EXTRACTED_WITH_LIB_EXTRACTION -gt $EXTRACTED_NORMAL ]]; then - echo "ERROR: EXTRACTED_WITH_LIB_EXTRACTION not greater than EXTRACTED_NORMAL" - exitcode=1 -fi - -exit $exitcode From ae8240a695b532778f8ef8f4864f626744cfc705 Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Tue, 9 Apr 2024 13:23:03 +0100 Subject: [PATCH 67/97] Delete models for JDK internal packages --- .../jdk.internal.access.foreign.model.yml | 10 - .../generated/jdk.internal.access.model.yml | 243 ------------------ .../ext/generated/jdk.internal.misc.model.yml | 11 - .../jdk.internal.org.objectweb.asm.model.yml | 15 -- 4 files changed, 279 deletions(-) delete mode 100644 java/ql/lib/ext/generated/jdk.internal.access.foreign.model.yml delete mode 100644 java/ql/lib/ext/generated/jdk.internal.access.model.yml delete mode 100644 java/ql/lib/ext/generated/jdk.internal.misc.model.yml delete mode 100644 java/ql/lib/ext/generated/jdk.internal.org.objectweb.asm.model.yml diff --git a/java/ql/lib/ext/generated/jdk.internal.access.foreign.model.yml b/java/ql/lib/ext/generated/jdk.internal.access.foreign.model.yml deleted file mode 100644 index 5cfddbd9303..00000000000 --- a/java/ql/lib/ext/generated/jdk.internal.access.foreign.model.yml +++ /dev/null @@ -1,10 +0,0 @@ -# THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT. -extensions: - - addsTo: - pack: codeql/java-all - extensible: neutralModel - data: - - ["jdk.internal.access.foreign", "UnmapperProxy", "address", "()", "summary", "df-generated"] - - ["jdk.internal.access.foreign", "UnmapperProxy", "fileDescriptor", "()", "summary", "df-generated"] - - ["jdk.internal.access.foreign", "UnmapperProxy", "isSync", "()", "summary", "df-generated"] - - ["jdk.internal.access.foreign", "UnmapperProxy", "unmap", "()", "summary", "df-generated"] diff --git a/java/ql/lib/ext/generated/jdk.internal.access.model.yml b/java/ql/lib/ext/generated/jdk.internal.access.model.yml deleted file mode 100644 index 37d80349303..00000000000 --- a/java/ql/lib/ext/generated/jdk.internal.access.model.yml +++ /dev/null @@ -1,243 +0,0 @@ -# THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT. -extensions: - - addsTo: - pack: codeql/java-all - extensible: summaryModel - data: - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", True, "registerCleanup", "(FileDescriptor,PhantomCleanable)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaIOFilePermissionAccess", True, "newPermPlusAltPath", "(FilePermission)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaIOFilePermissionAccess", True, "newPermUsingAltPath", "(FilePermission)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaIORandomAccessFileAccess", True, "openAndDelete", "(File,String)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "addEnableNativeAccess", "(Module)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "addOpensToAllUnnamed", "(Module,Set,Set)", "", "Argument[1].Element", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "addOpensToAllUnnamed", "(Module,Set,Set)", "", "Argument[2].Element", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "createOrGetClassLoaderValueMap", "(ClassLoader)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "defineModule", "(ClassLoader,ModuleDescriptor,URI)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "defineModule", "(ClassLoader,ModuleDescriptor,URI)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "definePackage", "(ClassLoader,String,Module)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "definePackage", "(ClassLoader,String,Module)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "defineUnnamedModule", "(ClassLoader)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "getBytesNoRepl", "(String,Charset)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "getBytesUTF8NoRepl", "(String)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "getServicesCatalog", "(ModuleLayer)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "join", "(String,String,String,String[],int)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "join", "(String,String,String,String[],int)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "join", "(String,String,String,String[],int)", "", "Argument[2]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "join", "(String,String,String,String[],int)", "", "Argument[3].ArrayElement", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "layers", "(ClassLoader)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "layers", "(ModuleLayer)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "newStringNoRepl", "(byte[],Charset)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "newStringUTF8NoRepl", "(byte[],int,int)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "newThreadWithAcc", "(Runnable,AccessControlContext)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "newThreadWithAcc", "(Runnable,AccessControlContext)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "setCause", "(Throwable,Throwable)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "stringConcatHelper", "(String,MethodType)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "stringConcatHelper", "(String,MethodType)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "collectCoordinates", "(VarHandle,int,MethodHandle)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "dropCoordinates", "(VarHandle,int,Class[])", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "filterCoordinates", "(VarHandle,int,MethodHandle[])", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "filterValue", "(VarHandle,MethodHandle,MethodHandle)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "getMethodDescriptor", "(Object)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "getMethodType", "(Object)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "getName", "(Object)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "insertCoordinates", "(VarHandle,int,Object[])", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "nativeMethodHandle", "(NativeEntryPoint,MethodHandle)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "nativeMethodHandle", "(NativeEntryPoint,MethodHandle)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "permuteCoordinates", "(VarHandle,List,int[])", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "permuteCoordinates", "(VarHandle,List,int[])", "", "Argument[1].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newConfiguration", "(ModuleFinder,Map)", "", "Argument[1].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newExports", "(Set,String)", "", "Argument[0].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newExports", "(Set,String)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newExports", "(Set,String,Set)", "", "Argument[0].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newExports", "(Set,String,Set)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newExports", "(Set,String,Set)", "", "Argument[2].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleBuilder", "(String,boolean,Set)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleBuilder", "(String,boolean,Set)", "", "Argument[2].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[2].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[3].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[4].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[5].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[6].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[7].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[8].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[9]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newOpens", "(Set,String)", "", "Argument[0].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newOpens", "(Set,String)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newOpens", "(Set,String,Set)", "", "Argument[0].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newOpens", "(Set,String,Set)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newOpens", "(Set,String,Set)", "", "Argument[2].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newProvides", "(String,List)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newProvides", "(String,List)", "", "Argument[1].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newRequires", "(Set,String,ModuleDescriptor$Version)", "", "Argument[0].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newRequires", "(Set,String,ModuleDescriptor$Version)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newRequires", "(Set,String,ModuleDescriptor$Version)", "", "Argument[2]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "packages", "(ModuleDescriptor$Builder)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "copyConstructor", "(Constructor)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "copyField", "(Field)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "copyMethod", "(Method)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "getConstructorAccessor", "(Constructor)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "getConstructorAnnotations", "(Constructor)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "getConstructorParameterAnnotations", "(Constructor)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "getConstructorSignature", "(Constructor)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "getExecutableSharedParameterTypes", "(Executable)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "getMethodAccessor", "(Method)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "getRoot", "(AccessibleObject)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "leafCopyMethod", "(Method)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "newConstructor", "(Class,Class[],Class[],int,int,String,byte[],byte[])", "", "Argument[1].ArrayElement", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "newConstructor", "(Class,Class[],Class[],int,int,String,byte[],byte[])", "", "Argument[2].ArrayElement", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "newConstructor", "(Class,Class[],Class[],int,int,String,byte[],byte[])", "", "Argument[5]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "newConstructor", "(Class,Class[],Class[],int,int,String,byte[],byte[])", "", "Argument[6]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "newConstructor", "(Class,Class[],Class[],int,int,String,byte[],byte[])", "", "Argument[7]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "setConstructorAccessor", "(Constructor,ConstructorAccessor)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "setMethodAccessor", "(Method,MethodAccessor)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetHttpCookieAccess", True, "header", "(HttpCookie)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetHttpCookieAccess", True, "parse", "(String)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetInetAddressAccess", True, "addressBytes", "(Inet6Address)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetInetAddressAccess", True, "getByName", "(String,InetAddress)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetInetAddressAccess", True, "getOriginalHostName", "(InetAddress)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetURLAccess", True, "getHandler", "(URL)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetUriAccess", True, "create", "(String,String)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetUriAccess", True, "create", "(String,String)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "acquireScope", "(Buffer,boolean)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "bufferSegment", "(Buffer)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "getBufferBase", "(ByteBuffer)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "newDirectByteBuffer", "(long,int,Object,MemorySegmentProxy)", "", "Argument[2]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "newDirectByteBuffer", "(long,int,Object,MemorySegmentProxy)", "", "Argument[3]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "newHeapByteBuffer", "(byte[],int,int,MemorySegmentProxy)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "newHeapByteBuffer", "(byte[],int,int,MemorySegmentProxy)", "", "Argument[3]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "newMappedByteBuffer", "(UnmapperProxy,long,int,Object,MemorySegmentProxy)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "newMappedByteBuffer", "(UnmapperProxy,long,int,Object,MemorySegmentProxy)", "", "Argument[3]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "newMappedByteBuffer", "(UnmapperProxy,long,int,Object,MemorySegmentProxy)", "", "Argument[4]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecurityAccess$ProtectionDomainCache", True, "get", "(ProtectionDomain)", "", "Argument[this]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecurityAccess$ProtectionDomainCache", True, "put", "(ProtectionDomain,PermissionCollection)", "", "Argument[0]", "Argument[this]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecurityAccess$ProtectionDomainCache", True, "put", "(ProtectionDomain,PermissionCollection)", "", "Argument[1]", "Argument[this]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecurityAccess", True, "doIntersectionPrivilege", "(PrivilegedAction,AccessControlContext)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecurityAccess", True, "doIntersectionPrivilege", "(PrivilegedAction,AccessControlContext,AccessControlContext)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecurityAccess", True, "getProtectDomains", "(AccessControlContext)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecuritySignatureAccess", True, "initSign", "(Signature,PrivateKey,AlgorithmParameterSpec,SecureRandom)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecuritySignatureAccess", True, "initSign", "(Signature,PrivateKey,AlgorithmParameterSpec,SecureRandom)", "", "Argument[2]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecuritySignatureAccess", True, "initSign", "(Signature,PrivateKey,AlgorithmParameterSpec,SecureRandom)", "", "Argument[3]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecuritySignatureAccess", True, "initVerify", "(Signature,PublicKey,AlgorithmParameterSpec)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecuritySignatureAccess", True, "initVerify", "(Signature,PublicKey,AlgorithmParameterSpec)", "", "Argument[2]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilCollectionAccess", True, "listFromTrustedArray", "(Object[])", "", "Argument[0].ArrayElement", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilCollectionAccess", True, "listFromTrustedArrayNullsAllowed", "(Object[])", "", "Argument[0].ArrayElement", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "entryFor", "(JarFile,String)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "entryNames", "(JarFile,CodeSource[])", "", "Argument[1].ArrayElement", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getCodeSource", "(JarFile,URL,String)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getCodeSource", "(JarFile,URL,String)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getCodeSource", "(JarFile,URL,String)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getCodeSource", "(JarFile,URL,String)", "", "Argument[2]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getCodeSources", "(JarFile,URL)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getCodeSources", "(JarFile,URL)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getCodeSources", "(JarFile,URL)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getManifestDigests", "(JarFile)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getTrustedAttributes", "(Manifest,String)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilResourceBundleAccess", True, "getBundle", "(String,Locale,Module)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilResourceBundleAccess", True, "getBundle", "(String,Locale,Module)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilResourceBundleAccess", True, "getParent", "(ResourceBundle)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilResourceBundleAccess", True, "setLocale", "(ResourceBundle,Locale)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilResourceBundleAccess", True, "setName", "(ResourceBundle,String)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilResourceBundleAccess", True, "setParent", "(ResourceBundle,ResourceBundle)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - addsTo: - pack: codeql/java-all - extensible: neutralModel - data: - - ["jdk.internal.access", "JavaBeansAccess", "getConstructorPropertiesValue", "(Constructor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaBeansAccess", "getReadMethod", "(Class,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOAccess", "charset", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOAccess", "console", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "close", "(FileDescriptor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "get", "(FileDescriptor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "getAppend", "(FileDescriptor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "getHandle", "(FileDescriptor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "registerCleanup", "(FileDescriptor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "set", "(FileDescriptor,int)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "setAppend", "(FileDescriptor,boolean)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "setHandle", "(FileDescriptor,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "unregisterCleanup", "(FileDescriptor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addEnableNativeAccessAllUnnamed", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addExports", "(Module,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addExports", "(Module,String,Module)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addExportsToAllUnnamed", "(Module,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addNonExportedPackages", "(ModuleLayer)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addOpens", "(Module,String,Module)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addOpensToAllUnnamed", "(Module,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addReads", "(Module,Module)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addReadsAllUnnamed", "(Module)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addUses", "(Module,Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "bindToLoader", "(ModuleLayer,ClassLoader)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "blockedOn", "(Interruptible)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "casAnnotationType", "(Class,AnnotationType,AnnotationType)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "classData", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "decodeASCII", "(byte[],int,char[],int,int)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "defineClass", "(ClassLoader,Class,String,byte[],ProtectionDomain,boolean,int,Object)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "defineClass", "(ClassLoader,String,byte[],ProtectionDomain,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "exit", "(int)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "fastUUID", "(long,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "findBootstrapClassOrNull", "(String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "findNative", "(ClassLoader,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getAnnotationType", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getConstantPool", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getDeclaredAnnotationMap", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getDeclaredPublicMethods", "(Class,String,Class[])", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getEnumConstantsShared", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getRawClassAnnotations", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getRawClassTypeAnnotations", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getRawExecutableTypeAnnotations", "(Executable)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "inflateBytesToChars", "(byte[],int,char[],int,int)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "invalidatePackageAccessCache", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "invokeFinalize", "(Object)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "isEnableNativeAccess", "(Module)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "isReflectivelyExported", "(Module,String,Module)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "isReflectivelyOpened", "(Module,String,Module)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "protectionDomain", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "registerShutdownHook", "(int,boolean,Runnable)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "stringConcatInitialCoder", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "stringConcatMix", "(long,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", "ensureCustomized", "(MethodHandle)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", "generateHolderClasses", "(Stream)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", "getDeclaringClass", "(Object)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", "isNative", "(Object)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", "memoryAccessVarHandle", "(Class,boolean,long,ByteOrder)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", "newMemberName", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", "requires", "(ModuleDescriptor$Builder,Set,String,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", "resolveAndBind", "(ModuleFinder,Collection,PrintStream)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangRefAccess", "runFinalization", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangRefAccess", "waitForReferenceProcessing", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", "getConstructorSlot", "(Constructor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", "getExecutableTypeAnnotationBytes", "(Executable)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", "isTrustedFinalField", "(Field)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", "newInstance", "(Constructor,Object[],Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNetInetAddressAccess", "addressValue", "(Inet4Address)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "force", "(FileDescriptor,long,boolean,long,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "getBufferAddress", "(ByteBuffer)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "getDirectBufferPool", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "isLoaded", "(long,boolean,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "load", "(long,boolean,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "pageSize", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "reserveMemory", "(long,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "unload", "(long,boolean,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "unmapper", "(ByteBuffer)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "unreserveMemory", "(long,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaSecurityAccess", "getProtectionDomainCache", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaSecuritySignatureAccess", "initVerify", "(Signature,Certificate,AlgorithmParameterSpec)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaSecuritySpecAccess", "clearEncodedKeySpec", "(EncodedKeySpec)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", "ensureInitialization", "(JarFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", "entries2", "(JarFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", "isInitializing", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", "jarFileHasClassPathAttribute", "(JarFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", "setEagerValidation", "(JarFile,boolean)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilResourceBundleAccess", "newResourceBundle", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "entries", "(ZipFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "entryNameStream", "(ZipFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "getExtraAttributes", "(ZipEntry)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "getManifestAndSignatureRelatedFiles", "(JarFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "getManifestName", "(JarFile,boolean)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "getManifestNum", "(JarFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "getMetaInfVersions", "(JarFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "setExtraAttributes", "(ZipEntry,int)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "startsWithLocHeader", "(ZipFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "stream", "(ZipFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaxCryptoSpecAccess", "clearSecretKeySpec", "(SecretKeySpec)", "summary", "df-generated"] diff --git a/java/ql/lib/ext/generated/jdk.internal.misc.model.yml b/java/ql/lib/ext/generated/jdk.internal.misc.model.yml deleted file mode 100644 index 3ab4816a586..00000000000 --- a/java/ql/lib/ext/generated/jdk.internal.misc.model.yml +++ /dev/null @@ -1,11 +0,0 @@ -# THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT. -extensions: - - addsTo: - pack: codeql/java-all - extensible: neutralModel - data: - - ["jdk.internal.misc", "Signal$Handler", "handle", "(Signal)", "summary", "df-generated"] - - ["jdk.internal.misc", "VM$BufferPool", "getCount", "()", "summary", "df-generated"] - - ["jdk.internal.misc", "VM$BufferPool", "getMemoryUsed", "()", "summary", "df-generated"] - - ["jdk.internal.misc", "VM$BufferPool", "getName", "()", "summary", "df-generated"] - - ["jdk.internal.misc", "VM$BufferPool", "getTotalCapacity", "()", "summary", "df-generated"] diff --git a/java/ql/lib/ext/generated/jdk.internal.org.objectweb.asm.model.yml b/java/ql/lib/ext/generated/jdk.internal.org.objectweb.asm.model.yml deleted file mode 100644 index 1121dda6781..00000000000 --- a/java/ql/lib/ext/generated/jdk.internal.org.objectweb.asm.model.yml +++ /dev/null @@ -1,15 +0,0 @@ -# THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT. -extensions: - - addsTo: - pack: codeql/java-all - extensible: summaryModel - data: - - ["jdk.internal.org.objectweb.asm", "ClassVisitor", True, "visitAnnotation", "(String,boolean)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.org.objectweb.asm", "ClassVisitor", True, "visitAnnotation", "(String,boolean)", "", "Argument[this]", "ReturnValue", "taint", "df-generated"] - - addsTo: - pack: codeql/java-all - extensible: neutralModel - data: - - ["jdk.internal.org.objectweb.asm", "ClassVisitor", "visit", "(int,int,String,String,String,String[])", "summary", "df-generated"] - - ["jdk.internal.org.objectweb.asm", "ClassVisitor", "visitAttribute", "(Attribute)", "summary", "df-generated"] - - ["jdk.internal.org.objectweb.asm", "ClassVisitor", "visitModule", "(String,int,String)", "summary", "df-generated"] From e6d63b980d044c167d8cfa5f60117c3cbc82c633 Mon Sep 17 00:00:00 2001 From: Felicity Chapman Date: Tue, 9 Apr 2024 14:05:42 +0100 Subject: [PATCH 68/97] Make duplicated content clear --- .../customizing-library-models-for-csharp.rst | 15 +++------------ ...omizing-library-models-for-java-and-kotlin.rst | 11 +---------- .../codeql/reusables/threat-model-description.rst | 10 ++++++++++ 3 files changed, 14 insertions(+), 22 deletions(-) create mode 100644 docs/codeql/reusables/threat-model-description.rst diff --git a/docs/codeql/codeql-language-guides/customizing-library-models-for-csharp.rst b/docs/codeql/codeql-language-guides/customizing-library-models-for-csharp.rst index 09dcf36fc07..39b5ee30ee4 100644 --- a/docs/codeql/codeql-language-guides/customizing-library-models-for-csharp.rst +++ b/docs/codeql/codeql-language-guides/customizing-library-models-for-csharp.rst @@ -282,7 +282,7 @@ These are the same for both of the rows above as we are adding two summaries for - The second value ``Enumerable`` is the class (type) name. - The third value ``False`` is a flag that indicates whether or not the summary also applies to all overrides of the method. - The fourth value ``Select`` is the method name, along with the type parameters for the method. The names of the generic type parameters provided in the model must match the names of the generic type parameters in the method signature in the source code. -- The fifth value ``(System.Collections.Generic.IEnumerable,System.Func)`` is the method input type signature. The generics in the signature must match the generics in the method signature in the source code. +- The fifth value ``(System.Collections.Generic.IEnumerable,System.Func)`` is the method input type signature. The generics in the signature must match the generics in the method signature in the source code. The sixth value should be left empty and is out of scope for this documentation. The remaining values are used to define the ``access path``, the ``kind``, and the ``provenance`` (origin) of the summary definition. @@ -309,7 +309,7 @@ That is, the first row specifies that values can flow from the elements of the q Example: Add a ``neutral`` method ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This example shows how we can model a method as being neutral with respect to flow. We will also cover how to model a property by modeling the getter of the ``Now`` property of the ``DateTime`` class as neutral. +This example shows how we can model a method as being neutral with respect to flow. We will also cover how to model a property by modeling the getter of the ``Now`` property of the ``DateTime`` class as neutral. A neutral model is used to define that there is no flow through a method. .. code-block:: csharp @@ -346,13 +346,4 @@ The first four values identify the callable (in this case the getter of the ``No Threat models ------------- -.. include:: ../reusables/beta-note-threat-models.rst - -A threat model is a named class of dataflow sources that can be enabled or disabled independently. Threat models allow you to control the set of dataflow sources that you want to consider unsafe. For example, one codebase may only consider remote HTTP requests to be tainted, whereas another may also consider data from local files to be unsafe. You can use threat models to ensure that the relevant taint sources are used in a CodeQL analysis. - -The ``kind`` property of ``sourceModel`` determines which threat model a source is associated with. There are two main categories: - -- ``remote`` which represents requests and responses from the network. -- ``local`` which represents data from local files (``file``), command-line arguments (``commandargs``), database reads (``database``), and environment variables(``environment``). - -When running a CodeQL analysis, the ``remote`` threat model is included by default. You can optionally include other threat models as appropriate when using the CodeQL CLI and in GitHub code scanning. For more information, see `Analyzing your code with CodeQL queries `__ and `Customizing your advanced setup for code scanning `__. +.. include:: ../reusables/threat-model-description.rst diff --git a/docs/codeql/codeql-language-guides/customizing-library-models-for-java-and-kotlin.rst b/docs/codeql/codeql-language-guides/customizing-library-models-for-java-and-kotlin.rst index e8b69e20d81..dd7f352f6d0 100644 --- a/docs/codeql/codeql-language-guides/customizing-library-models-for-java-and-kotlin.rst +++ b/docs/codeql/codeql-language-guides/customizing-library-models-for-java-and-kotlin.rst @@ -297,13 +297,4 @@ The first four values identify the callable (in this case a method) to be modele Threat models ------------- -.. include:: ../reusables/beta-note-threat-models.rst - -A threat model is a named class of dataflow sources that can be enabled or disabled independently. Threat models allow you to control the set of dataflow sources that you want to consider unsafe. For example, one codebase may only consider remote HTTP requests to be tainted, whereas another may also consider data from local files to be unsafe. You can use threat models to ensure that the relevant taint sources are used in a CodeQL analysis. - -The ``kind`` property of the ``sourceModel`` determines which threat model a source is associated with. There are two main categories: - -- ``remote`` which represents requests and responses from the network. -- ``local`` which represents data from local files (``file``), command-line arguments (``commandargs``), database reads (``database``), and environment variables(``environment``). - -When running a CodeQL analysis, the ``remote`` threat model is included by default. You can optionally include other threat models as appropriate when using the CodeQL CLI and in GitHub code scanning. For more information, see `Analyzing your code with CodeQL queries `__ and `Customizing your advanced setup for code scanning `__. +.. include:: ../reusables/threat-model-description.rst diff --git a/docs/codeql/reusables/threat-model-description.rst b/docs/codeql/reusables/threat-model-description.rst new file mode 100644 index 00000000000..53a872487bf --- /dev/null +++ b/docs/codeql/reusables/threat-model-description.rst @@ -0,0 +1,10 @@ +.. include:: ../reusables/beta-note-threat-models.rst + +A threat model is a named class of dataflow sources that can be enabled or disabled independently. Threat models allow you to control the set of dataflow sources that you want to consider unsafe. For example, one codebase may only consider remote HTTP requests to be tainted, whereas another may also consider data from local files to be unsafe. You can use threat models to ensure that the relevant taint sources are used in a CodeQL analysis. + +The ``kind`` property of the ``sourceModel`` determines which threat model a source is associated with. There are two main categories: + +- ``remote`` which represents requests and responses from the network. +- ``local`` which represents data from local files (``file``), command-line arguments (``commandargs``), database reads (``database``), and environment variables(``environment``). + +When running a CodeQL analysis, the ``remote`` threat model is included by default. You can optionally include other threat models as appropriate when using the CodeQL CLI and in GitHub code scanning. For more information, see `Analyzing your code with CodeQL queries `__ and `Customizing your advanced setup for code scanning `__. From e821a62b44481a3ff06c74caec3a82f92ba546bc Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Fri, 5 Apr 2024 17:21:07 +0200 Subject: [PATCH 69/97] C++: Update expected test results --- cpp/ql/test/library-tests/CPP-205/elements.expected | 2 ++ cpp/ql/test/library-tests/ir/ir/PrintAST.expected | 10 ++++++++++ .../library-tests/ir/ir/operand_locations.expected | 4 ++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cpp/ql/test/library-tests/CPP-205/elements.expected b/cpp/ql/test/library-tests/CPP-205/elements.expected index 76104232401..9b60c25d650 100644 --- a/cpp/ql/test/library-tests/CPP-205/elements.expected +++ b/cpp/ql/test/library-tests/CPP-205/elements.expected @@ -10,7 +10,9 @@ | CPP-205.cpp:2:15:5:1 | { ... } | isFromUninstantiatedTemplate(fn) | | CPP-205.cpp:3:3:3:33 | declaration | isFromTemplateInstantiation(fn) | | CPP-205.cpp:3:3:3:33 | declaration | isFromUninstantiatedTemplate(fn) | +| CPP-205.cpp:3:15:3:15 | declaration of y | isFromTemplateInstantiation(fn) | | CPP-205.cpp:3:15:3:15 | declaration of y | isFromUninstantiatedTemplate(fn) | +| CPP-205.cpp:3:15:3:15 | y | isFromTemplateInstantiation(fn) | | CPP-205.cpp:3:15:3:15 | y | isFromUninstantiatedTemplate(fn) | | CPP-205.cpp:3:17:3:31 | 5 | isFromTemplateInstantiation(fn) | | CPP-205.cpp:4:3:4:11 | return ... | isFromTemplateInstantiation(fn) | diff --git a/cpp/ql/test/library-tests/ir/ir/PrintAST.expected b/cpp/ql/test/library-tests/ir/ir/PrintAST.expected index 7761598e90f..483bd43cdb4 100644 --- a/cpp/ql/test/library-tests/ir/ir/PrintAST.expected +++ b/cpp/ql/test/library-tests/ir/ir/PrintAST.expected @@ -15440,6 +15440,8 @@ ir.cpp: # 1911| Type = [CTypedefType,NestedTypedefType] pointer # 1911| getEntryPoint(): [BlockStmt] { ... } # 1912| getStmt(0): [DeclStmt] declaration +# 1912| getDeclarationEntry(0): [TypeDeclarationEntry] declaration of _Res +# 1912| Type = [CTypedefType,LocalTypedefType] _Res # 1913| getStmt(1): [ReturnStmt] return ... # 1913| getExpr(): [VariableAccess] p # 1913| Type = [CTypedefType,NestedTypedefType] pointer @@ -15547,6 +15549,10 @@ ir.cpp: # 1924| : # 1924| getEntryPoint(): [BlockStmt] { ... } # 1925| getStmt(0): [DeclStmt] declaration +# 1925| getDeclarationEntry(0): [VariableDeclarationEntry] definition of x +# 1925| Type = [ArrayType] int[10] +# 1925| getDeclarationEntry(1): [VariableDeclarationEntry] definition of y +# 1925| Type = [ArrayType] int[10] # 1926| getStmt(1): [ExprStmt] ExprStmt # 1926| getExpr(): [AssignExpr] ... = ... # 1926| Type = [IntType] int @@ -15642,7 +15648,11 @@ ir.cpp: # 1939| : # 1939| getEntryPoint(): [BlockStmt] { ... } # 1940| getStmt(0): [DeclStmt] declaration +# 1940| getDeclarationEntry(0): [VariableDeclarationEntry] declaration of g +# 1940| Type = [IntType] int # 1941| getStmt(1): [DeclStmt] declaration +# 1941| getDeclarationEntry(0): [FunctionDeclarationEntry] declaration of z +# 1941| Type = [IntType] int # 1942| getStmt(2): [ReturnStmt] return ... # 1942| getExpr(): [VariableAccess] g # 1942| Type = [IntType] int diff --git a/cpp/ql/test/library-tests/ir/ir/operand_locations.expected b/cpp/ql/test/library-tests/ir/ir/operand_locations.expected index 2799fc28e94..e331a6d53bb 100644 --- a/cpp/ql/test/library-tests/ir/ir/operand_locations.expected +++ b/cpp/ql/test/library-tests/ir/ir/operand_locations.expected @@ -9771,8 +9771,8 @@ | ir.cpp:1924:13:1924:13 | Load | m1928_9 | | ir.cpp:1924:13:1924:13 | SideEffect | m1924_3 | | ir.cpp:1924:13:1924:13 | SideEffect | m1924_8 | -| ir.cpp:1925:13:1925:29 | Address | &:r1925_1 | -| ir.cpp:1925:13:1925:29 | Address | &:r1925_3 | +| ir.cpp:1925:17:1925:17 | Address | &:r1925_1 | +| ir.cpp:1925:24:1925:24 | Address | &:r1925_3 | | ir.cpp:1926:13:1926:14 | Address | &:r1926_4 | | ir.cpp:1926:13:1926:19 | ChiPartial | partial:m1926_5 | | ir.cpp:1926:13:1926:19 | ChiTotal | total:m1925_2 | From 46c44b4dc0686e9751a9f63ce9fa6dcb369c671a Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Fri, 5 Apr 2024 19:37:45 +0200 Subject: [PATCH 70/97] C++: Update QLDoc of `IRDeclarationEntry` to reflect current reality --- .../cpp/ir/implementation/raw/internal/TranslatedElement.qll | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedElement.qll b/cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedElement.qll index d6ad6b8fde4..c02305dfc22 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedElement.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedElement.qll @@ -506,8 +506,7 @@ private module IRDeclarationEntries { * An entity that represents a declaration entry in the database. * * This class exists to work around the fact that `DeclStmt`s in some cases - * do not have `DeclarationEntry`s. Currently, this is the case for: - * - `DeclStmt`s in template instantiations. + * do not have `DeclarationEntry`s in older databases. * * So instead, the IR works with `IRDeclarationEntry`s that synthesize missing * `DeclarationEntry`s when there is no result for `DeclStmt::getDeclarationEntry`. From deb78b248b8d04aab1702c721d64db89de748a44 Mon Sep 17 00:00:00 2001 From: Max Schaefer <54907921+max-schaefer@users.noreply.github.com> Date: Tue, 9 Apr 2024 14:38:17 +0100 Subject: [PATCH 71/97] Apply suggestions from code review Co-authored-by: Taus --- .../test/AutomodelApplicationModeExtraction/Test.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java b/java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java index a8280bcaf27..9691cf86c15 100644 --- a/java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java +++ b/java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java @@ -103,10 +103,10 @@ class MoreTests { Files.delete( p // $ sinkModelCandidate=delete(Path):Argument[0] positiveSinkExample=delete(Path):Argument[0](path-injection) - ); // $ not a source candidate (return type is void) + ); // not a source candidate (return type is void) Files.deleteIfExists( p // $ sinkModelCandidate=deleteIfExists(Path):Argument[0] positiveSinkExample=deleteIfExists(Path):Argument[0](path-injection) - ); // $ not a source candidate (return type is boolean) + ); // not a source candidate (return type is boolean) } } \ No newline at end of file From 2a7420ce11ef59eca1d0228d972e534c3d3df8ec Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Tue, 9 Apr 2024 15:00:23 +0100 Subject: [PATCH 72/97] Update cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll Co-authored-by: Jeroen Ketema <93738568+jketema@users.noreply.github.com> --- cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll index 44f2e60daac..d8ce666a8cb 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll @@ -7,7 +7,7 @@ private module Input implements TypeFlowInput { // The address of a variable is always a single object i instanceof VariableAddressInstruction or - // A reference always points to a always a single object + // A reference always points to a single object i.getResultLanguageType().hasUnspecifiedType(any(ReferenceType rt), false) or // `this` is never an array From 291cc0a671a4bcb9d411f6b8016e61cb0209ad02 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Tue, 9 Apr 2024 15:25:13 +0100 Subject: [PATCH 73/97] C++: Anonymous namespaces provide internal linkage. --- cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll index d8ce666a8cb..ee493f00ce4 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll @@ -120,10 +120,13 @@ private module Input implements TypeFlowInput { * calls to this function. */ private predicate isPrivate(Function func) { + // static functions have internal linkage func.isStatic() or - func.getNamespace().getParentNamespace*().isInline() + // anonymous namespaces have internal linkage + func.getNamespace().getParentNamespace*().isAnonymous() or + // private member functions are only called internally from inside the class func.(MemberFunction).isPrivate() } From a53ef495ee937e58bfd401f47732a59887f1e06c Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Tue, 9 Apr 2024 16:04:20 +0100 Subject: [PATCH 74/97] C++: Simplify 'hasExactBufferType' and add comments. --- .../cpp/ir/dataflow/internal/TypeFlow.qll | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll index ee493f00ce4..69f94dad91b 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll @@ -3,6 +3,29 @@ private import semmle.code.cpp.ir.IR private import codeql.typeflow.TypeFlow private module Input implements TypeFlowInput { + /** Holds if `alloc` dynamically allocates a single object. */ + private predicate isSingleObjectAllocation(AllocationExpr alloc) { + // i.e., `new int`; + alloc instanceof NewExpr + or + // i.e., `malloc(sizeof(int))` + exists(SizeofTypeOperator sizeOf | sizeOf = alloc.getSizeExpr() | + not sizeOf.getTypeOperand().getUnspecifiedType() instanceof ArrayType + ) + } + + /** + * Holds if `i` is the result of a dynamic allocation. + * + * `isObject` is `true` if the allocation allocated a single object, + * and `false` otherwise. + */ + private predicate isAllocation(Instruction i, boolean isObject) { + exists(AllocationExpr alloc | alloc = i.getUnconvertedResultExpression() | + if isSingleObjectAllocation(alloc) then isObject = true else isObject = false + ) + } + private predicate hasExactSingleType(Instruction i) { // The address of a variable is always a single object i instanceof VariableAddressInstruction @@ -14,23 +37,16 @@ private module Input implements TypeFlowInput { i instanceof InitializeThisInstruction or // An allocation of a non-array object - exists(AllocationExpr alloc | alloc = i.getUnconvertedResultExpression() | - // i.e., `new int`; - alloc instanceof NewExpr - or - // i.e., `malloc(sizeof(int))` - exists(SizeofTypeOperator sizeOf | sizeOf = alloc.getSizeExpr() | - not sizeOf.getTypeOperand().getUnspecifiedType() instanceof ArrayType - ) - ) + isAllocation(i, true) } private predicate hasExactBufferType(Instruction i) { // Anything with an array type is a buffer i.getResultLanguageType().hasUnspecifiedType(any(ArrayType at), false) or - not hasExactSingleType(i) and - i.getUnconvertedResultExpression() instanceof AllocationExpr + // An allocation expression that we couldn't conclude allocated a single + // expression is assigned a buffer type. + isAllocation(i, false) } private newtype TTypeFlowNode = From b9cfeaf6140617109572152b84ab84fbf4be067e Mon Sep 17 00:00:00 2001 From: Dave Bartolomeo Date: Tue, 9 Apr 2024 12:41:32 -0400 Subject: [PATCH 75/97] Add test case --- java/ql/test/query-tests/SpuriousJavadocParam/Test.java | 1 + java/ql/test/query-tests/SpuriousJavadocParam/test.expected | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/java/ql/test/query-tests/SpuriousJavadocParam/Test.java b/java/ql/test/query-tests/SpuriousJavadocParam/Test.java index 434353f4c67..d8891afb756 100644 --- a/java/ql/test/query-tests/SpuriousJavadocParam/Test.java +++ b/java/ql/test/query-tests/SpuriousJavadocParam/Test.java @@ -128,6 +128,7 @@ public class Test { /** * @param exists + * @param does not * @param i exists * @param k does not */ diff --git a/java/ql/test/query-tests/SpuriousJavadocParam/test.expected b/java/ql/test/query-tests/SpuriousJavadocParam/test.expected index 0ade1a7d519..f184473ea4a 100644 --- a/java/ql/test/query-tests/SpuriousJavadocParam/test.expected +++ b/java/ql/test/query-tests/SpuriousJavadocParam/test.expected @@ -13,4 +13,5 @@ | Test.java:118:6:118:12 | @param | @param tag "T" does not match any actual type parameter of type "GenericInterface". | | Test.java:119:6:119:12 | @param | @param tag "" does not match any actual type parameter of type "GenericInterface". | | Test.java:125:6:125:12 | @param | @param tag "k" does not match any actual type parameter or record parameter of record "SomeRecord". | -| Test.java:132:6:132:12 | @param | @param tag "k" does not match any actual type parameter or record parameter of record "GenericRecord". | +| Test.java:131:6:131:12 | @param | @param tag "" does not match any actual type parameter or record parameter of record "GenericRecord". | +| Test.java:133:6:133:12 | @param | @param tag "k" does not match any actual type parameter or record parameter of record "GenericRecord". | From 4fed3cf12d9145b2d891009ff409853da9a61ebb Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 10 Apr 2024 11:31:34 +0200 Subject: [PATCH 76/97] Python: Fix `RemoteFlowSourceFromCsv` --- python/ql/lib/semmle/python/frameworks/data/ModelsAsData.qll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ql/lib/semmle/python/frameworks/data/ModelsAsData.qll b/python/ql/lib/semmle/python/frameworks/data/ModelsAsData.qll index 34e48439271..2f4c74ea9e4 100644 --- a/python/ql/lib/semmle/python/frameworks/data/ModelsAsData.qll +++ b/python/ql/lib/semmle/python/frameworks/data/ModelsAsData.qll @@ -22,7 +22,7 @@ private import semmle.python.dataflow.new.FlowSummary /** * A remote flow source originating from a CSV source row. */ -private class RemoteFlowSourceFromCsv extends RemoteFlowSource { +private class RemoteFlowSourceFromCsv extends RemoteFlowSource::Range { RemoteFlowSourceFromCsv() { this = ModelOutput::getASourceNode("remote").asSource() } override string getSourceType() { result = "Remote flow (from model)" } From b4829addf7b3c2ac0121e6dc0222d87381a166cc Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Wed, 10 Apr 2024 10:48:23 +0100 Subject: [PATCH 77/97] Add comments with link to online doc --- go/extractor/extractor.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/go/extractor/extractor.go b/go/extractor/extractor.go index 1a60c2bf982..acfccf33e3c 100644 --- a/go/extractor/extractor.go +++ b/go/extractor/extractor.go @@ -388,6 +388,8 @@ func extractUniverseScope() { } // extractObjects extracts all objects declared in the given scope +// For more information on objects, see: +// https://github.com/golang/example/blob/master/gotypes/README.md#objects func extractObjects(tw *trap.Writer, scope *types.Scope, scopeLabel trap.Label) { for _, name := range scope.Names() { obj := scope.Lookup(name) @@ -440,6 +442,8 @@ func extractMethod(tw *trap.Writer, meth *types.Func) trap.Label { } // extractObject extracts a single object and emits it to the objects table. +// For more information on objects, see: +// https://github.com/golang/example/blob/master/gotypes/README.md#objects func extractObject(tw *trap.Writer, obj types.Object, lbl trap.Label) { name := obj.Name() isBuiltin := obj.Parent() == types.Universe @@ -487,6 +491,8 @@ func extractObject(tw *trap.Writer, obj types.Object, lbl trap.Label) { } // extractObjectTypes extracts type and receiver information for all objects +// For more information on objects, see: +// https://github.com/golang/example/blob/master/gotypes/README.md#objects func extractObjectTypes(tw *trap.Writer) { // calling `extractType` on a named type will extract all methods defined // on it, which will add new objects. Therefore we need to do this first @@ -497,11 +503,13 @@ func extractObjectTypes(tw *trap.Writer) { } changed = tw.ForEachObject(emitObjectType) if changed { - log.Printf("Warning: more objects were labeled while emitted object types") + log.Printf("Warning: more objects were labeled while emitting object types") } } // extractObjectType extracts type and receiver information for a given object +// For more information on objects, see: +// https://github.com/golang/example/blob/master/gotypes/README.md#objects func extractObjectType(tw *trap.Writer, obj types.Object, lbl trap.Label) { if tp := obj.Type(); tp != nil { extractType(tw, tp) From bee54e42479867a901e546d4782f3f80f73a1db8 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Fri, 5 Apr 2024 14:41:10 +0200 Subject: [PATCH 78/97] C#: Add desktop runtime dependencies integration test. --- .../Assemblies.expected | 210 ++++++++++++++++++ .../Assemblies.ql | 21 ++ .../Program.cs | 6 + .../global.json | 5 + .../test.py | 3 + 5 files changed, 245 insertions(+) create mode 100644 csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.expected create mode 100644 csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.ql create mode 100644 csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Program.cs create mode 100644 csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/global.json create mode 100644 csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/test.py diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.expected b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.expected new file mode 100644 index 00000000000..f4f8290eb75 --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.expected @@ -0,0 +1,210 @@ +| [...]/Humanizer.dll | +| [...]/Microsoft.Bcl.AsyncInterfaces.dll | +| [...]/Microsoft.Build.Framework.dll | +| [...]/Microsoft.Build.dll | +| [...]/Microsoft.CSharp.dll | +| [...]/Microsoft.CodeAnalysis.CSharp.Workspaces.dll | +| [...]/Microsoft.CodeAnalysis.CSharp.dll | +| [...]/Microsoft.CodeAnalysis.VisualBasic.Workspaces.dll | +| [...]/Microsoft.CodeAnalysis.VisualBasic.dll | +| [...]/Microsoft.CodeAnalysis.Workspaces.dll | +| [...]/Microsoft.CodeAnalysis.dll | +| [...]/Microsoft.NET.StringTools.dll | +| [...]/Microsoft.VisualBasic.Core.dll | +| [...]/Microsoft.VisualBasic.dll | +| [...]/Microsoft.Win32.Primitives.dll | +| [...]/Microsoft.Win32.Registry.dll | +| [...]/Microsoft.Win32.SystemEvents.dll | +| [...]/Mono.Posix.NETStandard.dll | +| [...]/Newtonsoft.Json.dll | +| [...]/Semmle.Autobuild.CSharp.dll | +| [...]/Semmle.Autobuild.Shared.dll | +| [...]/Semmle.Extraction.CSharp.DependencyFetching.dll | +| [...]/Semmle.Extraction.CSharp.Driver.dll | +| [...]/Semmle.Extraction.CSharp.Standalone.dll | +| [...]/Semmle.Extraction.CSharp.Util.dll | +| [...]/Semmle.Extraction.CSharp.dll | +| [...]/Semmle.Extraction.dll | +| [...]/Semmle.Util.dll | +| [...]/System.AppContext.dll | +| [...]/System.Buffers.dll | +| [...]/System.Collections.Concurrent.dll | +| [...]/System.Collections.Immutable.dll | +| [...]/System.Collections.NonGeneric.dll | +| [...]/System.Collections.Specialized.dll | +| [...]/System.Collections.dll | +| [...]/System.ComponentModel.Annotations.dll | +| [...]/System.ComponentModel.DataAnnotations.dll | +| [...]/System.ComponentModel.EventBasedAsync.dll | +| [...]/System.ComponentModel.Primitives.dll | +| [...]/System.ComponentModel.TypeConverter.dll | +| [...]/System.ComponentModel.dll | +| [...]/System.Composition.AttributedModel.dll | +| [...]/System.Composition.Convention.dll | +| [...]/System.Composition.Hosting.dll | +| [...]/System.Composition.Runtime.dll | +| [...]/System.Composition.TypedParts.dll | +| [...]/System.Configuration.ConfigurationManager.dll | +| [...]/System.Configuration.dll | +| [...]/System.Console.dll | +| [...]/System.Core.dll | +| [...]/System.Data.Common.dll | +| [...]/System.Data.DataSetExtensions.dll | +| [...]/System.Data.dll | +| [...]/System.Diagnostics.Contracts.dll | +| [...]/System.Diagnostics.Debug.dll | +| [...]/System.Diagnostics.DiagnosticSource.dll | +| [...]/System.Diagnostics.EventLog.dll | +| [...]/System.Diagnostics.FileVersionInfo.dll | +| [...]/System.Diagnostics.Process.dll | +| [...]/System.Diagnostics.StackTrace.dll | +| [...]/System.Diagnostics.TextWriterTraceListener.dll | +| [...]/System.Diagnostics.Tools.dll | +| [...]/System.Diagnostics.TraceSource.dll | +| [...]/System.Diagnostics.Tracing.dll | +| [...]/System.Drawing.Common.dll | +| [...]/System.Drawing.Primitives.dll | +| [...]/System.Drawing.dll | +| [...]/System.Dynamic.Runtime.dll | +| [...]/System.Formats.Asn1.dll | +| [...]/System.Formats.Tar.dll | +| [...]/System.Globalization.Calendars.dll | +| [...]/System.Globalization.Extensions.dll | +| [...]/System.Globalization.dll | +| [...]/System.IO.Compression.Brotli.dll | +| [...]/System.IO.Compression.FileSystem.dll | +| [...]/System.IO.Compression.ZipFile.dll | +| [...]/System.IO.Compression.dll | +| [...]/System.IO.FileSystem.AccessControl.dll | +| [...]/System.IO.FileSystem.DriveInfo.dll | +| [...]/System.IO.FileSystem.Primitives.dll | +| [...]/System.IO.FileSystem.Watcher.dll | +| [...]/System.IO.FileSystem.dll | +| [...]/System.IO.IsolatedStorage.dll | +| [...]/System.IO.MemoryMappedFiles.dll | +| [...]/System.IO.Pipelines.dll | +| [...]/System.IO.Pipes.AccessControl.dll | +| [...]/System.IO.Pipes.dll | +| [...]/System.IO.UnmanagedMemoryStream.dll | +| [...]/System.IO.dll | +| [...]/System.Linq.Expressions.dll | +| [...]/System.Linq.Parallel.dll | +| [...]/System.Linq.Queryable.dll | +| [...]/System.Linq.dll | +| [...]/System.Memory.dll | +| [...]/System.Net.Http.Json.dll | +| [...]/System.Net.Http.dll | +| [...]/System.Net.HttpListener.dll | +| [...]/System.Net.Mail.dll | +| [...]/System.Net.NameResolution.dll | +| [...]/System.Net.NetworkInformation.dll | +| [...]/System.Net.Ping.dll | +| [...]/System.Net.Primitives.dll | +| [...]/System.Net.Quic.dll | +| [...]/System.Net.Requests.dll | +| [...]/System.Net.Security.dll | +| [...]/System.Net.ServicePoint.dll | +| [...]/System.Net.Sockets.dll | +| [...]/System.Net.WebClient.dll | +| [...]/System.Net.WebHeaderCollection.dll | +| [...]/System.Net.WebProxy.dll | +| [...]/System.Net.WebSockets.Client.dll | +| [...]/System.Net.WebSockets.dll | +| [...]/System.Net.dll | +| [...]/System.Numerics.Vectors.dll | +| [...]/System.Numerics.dll | +| [...]/System.ObjectModel.dll | +| [...]/System.Private.CoreLib.dll | +| [...]/System.Private.DataContractSerialization.dll | +| [...]/System.Private.Uri.dll | +| [...]/System.Private.Xml.Linq.dll | +| [...]/System.Private.Xml.dll | +| [...]/System.Reflection.DispatchProxy.dll | +| [...]/System.Reflection.Emit.ILGeneration.dll | +| [...]/System.Reflection.Emit.Lightweight.dll | +| [...]/System.Reflection.Emit.dll | +| [...]/System.Reflection.Extensions.dll | +| [...]/System.Reflection.Metadata.dll | +| [...]/System.Reflection.MetadataLoadContext.dll | +| [...]/System.Reflection.Primitives.dll | +| [...]/System.Reflection.TypeExtensions.dll | +| [...]/System.Reflection.dll | +| [...]/System.Resources.Reader.dll | +| [...]/System.Resources.ResourceManager.dll | +| [...]/System.Resources.Writer.dll | +| [...]/System.Runtime.CompilerServices.Unsafe.dll | +| [...]/System.Runtime.CompilerServices.VisualC.dll | +| [...]/System.Runtime.Extensions.dll | +| [...]/System.Runtime.Handles.dll | +| [...]/System.Runtime.InteropServices.JavaScript.dll | +| [...]/System.Runtime.InteropServices.RuntimeInformation.dll | +| [...]/System.Runtime.InteropServices.dll | +| [...]/System.Runtime.Intrinsics.dll | +| [...]/System.Runtime.Loader.dll | +| [...]/System.Runtime.Numerics.dll | +| [...]/System.Runtime.Serialization.Formatters.dll | +| [...]/System.Runtime.Serialization.Json.dll | +| [...]/System.Runtime.Serialization.Primitives.dll | +| [...]/System.Runtime.Serialization.Xml.dll | +| [...]/System.Runtime.Serialization.dll | +| [...]/System.Runtime.dll | +| [...]/System.Security.AccessControl.dll | +| [...]/System.Security.Claims.dll | +| [...]/System.Security.Cryptography.Algorithms.dll | +| [...]/System.Security.Cryptography.Cng.dll | +| [...]/System.Security.Cryptography.Csp.dll | +| [...]/System.Security.Cryptography.Encoding.dll | +| [...]/System.Security.Cryptography.OpenSsl.dll | +| [...]/System.Security.Cryptography.Primitives.dll | +| [...]/System.Security.Cryptography.ProtectedData.dll | +| [...]/System.Security.Cryptography.X509Certificates.dll | +| [...]/System.Security.Cryptography.dll | +| [...]/System.Security.Permissions.dll | +| [...]/System.Security.Principal.Windows.dll | +| [...]/System.Security.Principal.dll | +| [...]/System.Security.SecureString.dll | +| [...]/System.Security.dll | +| [...]/System.ServiceModel.Web.dll | +| [...]/System.ServiceProcess.dll | +| [...]/System.Text.Encoding.CodePages.dll | +| [...]/System.Text.Encoding.Extensions.dll | +| [...]/System.Text.Encoding.dll | +| [...]/System.Text.Encodings.Web.dll | +| [...]/System.Text.Json.dll | +| [...]/System.Text.RegularExpressions.dll | +| [...]/System.Threading.Channels.dll | +| [...]/System.Threading.Overlapped.dll | +| [...]/System.Threading.Tasks.Dataflow.dll | +| [...]/System.Threading.Tasks.Extensions.dll | +| [...]/System.Threading.Tasks.Parallel.dll | +| [...]/System.Threading.Tasks.dll | +| [...]/System.Threading.Thread.dll | +| [...]/System.Threading.ThreadPool.dll | +| [...]/System.Threading.Timer.dll | +| [...]/System.Threading.dll | +| [...]/System.Transactions.Local.dll | +| [...]/System.Transactions.dll | +| [...]/System.ValueTuple.dll | +| [...]/System.Web.HttpUtility.dll | +| [...]/System.Web.dll | +| [...]/System.Windows.Extensions.dll | +| [...]/System.Windows.dll | +| [...]/System.Xml.Linq.dll | +| [...]/System.Xml.ReaderWriter.dll | +| [...]/System.Xml.Serialization.dll | +| [...]/System.Xml.XDocument.dll | +| [...]/System.Xml.XPath.XDocument.dll | +| [...]/System.Xml.XPath.dll | +| [...]/System.Xml.XmlDocument.dll | +| [...]/System.Xml.XmlSerializer.dll | +| [...]/System.Xml.dll | +| [...]/System.dll | +| [...]/WindowsBase.dll | +| [...]/mscorlib.dll | +| [...]/netstandard.dll | +| [...]/zh-Hant/Microsoft.CodeAnalysis.CSharp.Workspaces.resources.dll | +| [...]/zh-Hant/Microsoft.CodeAnalysis.CSharp.resources.dll | +| [...]/zh-Hant/Microsoft.CodeAnalysis.VisualBasic.Workspaces.resources.dll | +| [...]/zh-Hant/Microsoft.CodeAnalysis.VisualBasic.resources.dll | +| [...]/zh-Hant/Microsoft.CodeAnalysis.Workspaces.resources.dll | +| [...]/zh-Hant/Microsoft.CodeAnalysis.resources.dll | diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.ql b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.ql new file mode 100644 index 00000000000..3bb648c579a --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.ql @@ -0,0 +1,21 @@ +import csharp + +private string getPath(Assembly a) { + not a.getCompilation().getOutputAssembly() = a and + exists(string s | s = a.getFile().getAbsolutePath() | + result = + "[...]" + + s.substring(s.indexOf("test-db/working/") + "test-db/working/".length() + 16 + + "/packages".length(), s.length()) + or + exists(string sub | sub = "csharp/tools/" + ["osx64", "linux64"] | + result = "[...]" + s.substring(s.indexOf(sub) + sub.length(), s.length()) + ) + or + result = s and + not exists(s.indexOf(["test-db/working/", "csharp/tools/"])) + ) +} + +from Assembly a +select getPath(a) diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Program.cs b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Program.cs new file mode 100644 index 00000000000..39a9e95bb6e --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Program.cs @@ -0,0 +1,6 @@ +class Program +{ + static void Main(string[] args) + { + } +} \ No newline at end of file diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/global.json b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/global.json new file mode 100644 index 00000000000..d54915e8d4d --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/global.json @@ -0,0 +1,5 @@ +{ + "sdk": { + "version": "8.0.101" + } +} diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/test.py b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/test.py new file mode 100644 index 00000000000..a17966e148a --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/test.py @@ -0,0 +1,3 @@ +from create_database_utils import * + +run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"]) From 6299d9cecd1e562c9859a2ee8b0f3580ed391ff1 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Wed, 20 Mar 2024 15:19:13 +0100 Subject: [PATCH 79/97] C#: Introduce AssemblyPath and re-factor AssemblyCache to use this instead of strings. --- .../AssemblyCache.cs | 108 +++++++++++++----- .../DependencyManager.Nuget.cs | 8 +- .../DependencyManager.cs | 22 ++-- 3 files changed, 92 insertions(+), 46 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs index 3d6672560b6..0d2393faa0a 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs @@ -1,10 +1,84 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.IO; using System.Linq; using Semmle.Util.Logging; namespace Semmle.Extraction.CSharp.DependencyFetching { + /// + /// Used to represent a path to an assembly or a directory containing assemblies + /// and a selector function to determine which files to include, when indexing the assemblies. + /// + internal sealed class AssemblyPath(string p, Func includeFileName) + { + public string Path => p; + + public AssemblyPath(string p) : this(p, _ => true) { } + + public static implicit operator AssemblyPath(string path) => new(path); + + /// + /// Finds all assemblies nested within the directory `path` + /// and adds them to the a list of assembly names to index. + /// Indexing is performed at a later stage. This only collects the names. + /// + /// The directory to index. + private void AddReferenceDirectory(List dllsToIndex, ILogger logger) + { + foreach (var dll in new DirectoryInfo(p).EnumerateFiles("*.dll", SearchOption.AllDirectories)) + { + if (includeFileName(dll.Name)) + { + dllsToIndex.Add(dll.FullName); + } + else + { + logger.LogInfo($"AssemblyPath: Skipping {dll.FullName}."); + } + } + } + + /// + /// Finds all assemblies in `p` that should be indexed and adds them to + /// the list of assembly names to index. + /// + /// List of assembly names to index. + /// Logger + public void Process(List dllsToIndex, ILogger logger) + { + if (File.Exists(p)) + { + if (includeFileName(System.IO.Path.GetFileName(p))) + { + dllsToIndex.Add(p); + } + else + { + logger.LogInfo($"AssemblyPath: Skipping {p}."); + } + return; + } + + if (Directory.Exists(p)) + { + logger.LogInfo($"AssemblyPath: Finding reference DLLs in {p}..."); + AddReferenceDirectory(dllsToIndex, logger); + } + else + { + logger.LogInfo("AssemblyCache: Path not found: " + p); + } + } + + public override bool Equals(object? obj) => + obj is AssemblyPath ap && p.Equals(ap.Path); + + public override int GetHashCode() => p.GetHashCode(); + + public override string ToString() => p; + } + /// /// Manages the set of assemblies. /// Searches for assembly DLLs, indexes them and provides @@ -20,44 +94,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// assembly cache. /// /// Callback for progress. - public AssemblyCache(IEnumerable paths, IEnumerable frameworkPaths, ILogger logger) + public AssemblyCache(IEnumerable paths, IEnumerable frameworkPaths, ILogger logger) { this.logger = logger; foreach (var path in paths) { - if (File.Exists(path)) - { - dllsToIndex.Add(path); - continue; - } - - if (Directory.Exists(path)) - { - logger.LogInfo($"Finding reference DLLs in {path}..."); - AddReferenceDirectory(path); - } - else - { - logger.LogInfo("AssemblyCache: Path not found: " + path); - } + path.Process(dllsToIndex, logger); } IndexReferences(frameworkPaths); } - /// - /// Finds all assemblies nested within a directory - /// and adds them to its index. - /// (Indexing is performed at a later stage by IndexReferences()). - /// - /// The directory to index. - private void AddReferenceDirectory(string dir) - { - foreach (var dll in new DirectoryInfo(dir).EnumerateFiles("*.dll", SearchOption.AllDirectories)) - { - dllsToIndex.Add(dll.FullName); - } - } - /// /// Indexes all DLLs we have located. /// Because this is a potentially time-consuming operation, it is put into a separate stage. diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs index 1f3f44425d5..e808235e54d 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs @@ -12,7 +12,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { public sealed partial class DependencyManager { - private void RestoreNugetPackages(List allNonBinaryFiles, IEnumerable allProjects, IEnumerable allSolutions, HashSet dllPaths) + private void RestoreNugetPackages(List allNonBinaryFiles, IEnumerable allProjects, IEnumerable allSolutions, HashSet dllPaths) { try { @@ -55,7 +55,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } nugetPackageDllPaths.ExceptWith(excludedPaths); - dllPaths.UnionWith(nugetPackageDllPaths); + dllPaths.UnionWith(nugetPackageDllPaths.Select(p => new AssemblyPath(p))); } catch (Exception exc) { @@ -72,7 +72,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching .Paths .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d)) .ToList(); - dllPaths.UnionWith(paths); + dllPaths.UnionWith(paths.Select(p => new AssemblyPath(p))); LogAllUnusedPackages(dependencies); DownloadMissingPackages(allNonBinaryFiles, dllPaths); @@ -148,7 +148,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString())); } - private void DownloadMissingPackages(List allFiles, ISet dllPaths, bool withNugetConfig = true) + private void DownloadMissingPackages(List allFiles, ISet dllPaths, bool withNugetConfig = true) { var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo); var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index 8d63d7f42b2..55c41f31601 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -46,7 +46,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// /// Performs C# dependency fetching. /// - /// Dependency fetching options + /// Path to directory containing source code. /// Logger for dependency fetching progress. public DependencyManager(string srcDir, ILogger logger) { @@ -91,7 +91,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching this.generatedSources = new(); var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj").ToList(); var allSolutions = allNonBinaryFiles.SelectFileNamesByExtension(".sln").ToList(); - var dllPaths = allFiles.SelectFileNamesByExtension(".dll").ToHashSet(); + var dllPaths = allFiles.SelectFileNamesByExtension(".dll").Select(x => x).ToHashSet(); logger.LogInfo($"Found {allFiles.Count} files, {nonGeneratedSources.Count} source files, {allProjects.Count} project files, {allSolutions.Count} solution files, {dllPaths.Count} DLLs."); @@ -192,7 +192,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching ]); } - private HashSet AddFrameworkDlls(HashSet dllPaths) + private HashSet AddFrameworkDlls(HashSet dllPaths) { var frameworkLocations = new HashSet(); @@ -230,7 +230,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching continue; } - dllPaths.UnionWith(dlls); + dllPaths.UnionWith(dlls.Select(x => x)); frameworkLocations.UnionWith(dlls); } catch (Exception e) @@ -284,7 +284,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet dllPaths, ISet frameworkLocations) + private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet dllPaths, ISet frameworkLocations) { var versionFolders = GetPackageVersionSubDirectories(frameworkPath); if (versionFolders.Length > 1) @@ -313,7 +313,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching .ToArray(); } - private void RemoveFrameworkNugetPackages(ISet dllPaths, int fromIndex = 0) + private void RemoveFrameworkNugetPackages(ISet dllPaths, int fromIndex = 0) { var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks; for (var i = fromIndex; i < packagesInPrioOrder.Length; i++) @@ -322,7 +322,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - private void AddNetFrameworkDlls(ISet dllPaths, ISet frameworkLocations) + private void AddNetFrameworkDlls(ISet dllPaths, ISet frameworkLocations) { // Multiple dotnet framework packages could be present. // The order of the packages is important, we're adding the first one that is present in the nuget cache. @@ -375,7 +375,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching frameworkLocations.Add(runtimeLocation); } - private void RemoveNugetPackageReference(string packagePrefix, ISet dllPaths) + private void RemoveNugetPackageReference(string packagePrefix, ISet dllPaths) { var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant(); if (packageFolder == null) @@ -384,7 +384,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } var packagePathPrefix = Path.Combine(packageFolder, packagePrefix.ToLowerInvariant()); - var toRemove = dllPaths.Where(s => s.StartsWith(packagePathPrefix, StringComparison.InvariantCultureIgnoreCase)); + var toRemove = dllPaths.Where(s => s.Path.StartsWith(packagePathPrefix, StringComparison.InvariantCultureIgnoreCase)); foreach (var path in toRemove) { dllPaths.Remove(path); @@ -397,7 +397,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return fileContent.IsNewProjectStructureUsed && fileContent.UseAspNetCoreDlls; } - private void AddAspNetCoreFrameworkDlls(ISet dllPaths, ISet frameworkLocations) + private void AddAspNetCoreFrameworkDlls(ISet dllPaths, ISet frameworkLocations) { if (!IsAspNetCoreDetected()) { @@ -419,7 +419,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - private void AddMicrosoftWindowsDesktopDlls(ISet dllPaths, ISet frameworkLocations) + private void AddMicrosoftWindowsDesktopDlls(ISet dllPaths, ISet frameworkLocations) { if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework, packageDirectory) is string windowsDesktopApp) { From 646b272b4e0befe19902a10c3dee5af2e6870836 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Fri, 5 Apr 2024 10:01:20 +0200 Subject: [PATCH 80/97] C#: Move the AssemblyPath class to its own file. --- .../AssemblyCache.cs | 77 +----------------- .../AssemblyPath.cs | 80 +++++++++++++++++++ 2 files changed, 81 insertions(+), 76 deletions(-) create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyPath.cs diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs index 0d2393faa0a..f1ee65319ab 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs @@ -1,84 +1,9 @@ -using System; -using System.Collections.Generic; -using System.IO; +using System.Collections.Generic; using System.Linq; using Semmle.Util.Logging; namespace Semmle.Extraction.CSharp.DependencyFetching { - /// - /// Used to represent a path to an assembly or a directory containing assemblies - /// and a selector function to determine which files to include, when indexing the assemblies. - /// - internal sealed class AssemblyPath(string p, Func includeFileName) - { - public string Path => p; - - public AssemblyPath(string p) : this(p, _ => true) { } - - public static implicit operator AssemblyPath(string path) => new(path); - - /// - /// Finds all assemblies nested within the directory `path` - /// and adds them to the a list of assembly names to index. - /// Indexing is performed at a later stage. This only collects the names. - /// - /// The directory to index. - private void AddReferenceDirectory(List dllsToIndex, ILogger logger) - { - foreach (var dll in new DirectoryInfo(p).EnumerateFiles("*.dll", SearchOption.AllDirectories)) - { - if (includeFileName(dll.Name)) - { - dllsToIndex.Add(dll.FullName); - } - else - { - logger.LogInfo($"AssemblyPath: Skipping {dll.FullName}."); - } - } - } - - /// - /// Finds all assemblies in `p` that should be indexed and adds them to - /// the list of assembly names to index. - /// - /// List of assembly names to index. - /// Logger - public void Process(List dllsToIndex, ILogger logger) - { - if (File.Exists(p)) - { - if (includeFileName(System.IO.Path.GetFileName(p))) - { - dllsToIndex.Add(p); - } - else - { - logger.LogInfo($"AssemblyPath: Skipping {p}."); - } - return; - } - - if (Directory.Exists(p)) - { - logger.LogInfo($"AssemblyPath: Finding reference DLLs in {p}..."); - AddReferenceDirectory(dllsToIndex, logger); - } - else - { - logger.LogInfo("AssemblyCache: Path not found: " + p); - } - } - - public override bool Equals(object? obj) => - obj is AssemblyPath ap && p.Equals(ap.Path); - - public override int GetHashCode() => p.GetHashCode(); - - public override string ToString() => p; - } - /// /// Manages the set of assemblies. /// Searches for assembly DLLs, indexes them and provides diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyPath.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyPath.cs new file mode 100644 index 00000000000..816e0fef6a7 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyPath.cs @@ -0,0 +1,80 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Semmle.Util.Logging; + +namespace Semmle.Extraction.CSharp.DependencyFetching +{ + /// + /// Used to represent a path to an assembly or a directory containing assemblies + /// and a selector function to determine which files to include, when indexing the assemblies. + /// + internal sealed class AssemblyPath(string p, Func includeFileName) + { + public string Path => p; + + public AssemblyPath(string p) : this(p, _ => true) { } + + public static implicit operator AssemblyPath(string path) => new(path); + + /// + /// Finds all assemblies nested within the directory `path` + /// and adds them to the a list of assembly names to index. + /// Indexing is performed at a later stage. This only collects the names. + /// + /// The directory to index. + private void AddReferenceDirectory(List dllsToIndex, ILogger logger) + { + foreach (var dll in new DirectoryInfo(p).EnumerateFiles("*.dll", SearchOption.AllDirectories)) + { + if (includeFileName(dll.Name)) + { + dllsToIndex.Add(dll.FullName); + } + else + { + logger.LogInfo($"AssemblyPath: Skipping {dll.FullName}."); + } + } + } + + /// + /// Finds all assemblies in `p` that should be indexed and adds them to + /// the list of assembly names to index. + /// + /// List of assembly names to index. + /// Logger + public void Process(List dllsToIndex, ILogger logger) + { + if (File.Exists(p)) + { + if (includeFileName(System.IO.Path.GetFileName(p))) + { + dllsToIndex.Add(p); + } + else + { + logger.LogInfo($"AssemblyPath: Skipping {p}."); + } + return; + } + + if (Directory.Exists(p)) + { + logger.LogInfo($"AssemblyPath: Finding reference DLLs in {p}..."); + AddReferenceDirectory(dllsToIndex, logger); + } + else + { + logger.LogInfo("AssemblyCache: Path not found: " + p); + } + } + + public override bool Equals(object? obj) => + obj is AssemblyPath ap && p.Equals(ap.Path); + + public override int GetHashCode() => p.GetHashCode(); + + public override string ToString() => p; + } +} From d04bf6b6d637055103b795c00bbd9872c7276c96 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Fri, 5 Apr 2024 14:55:55 +0200 Subject: [PATCH 81/97] C#: Don't include Semmle.* dlls, if the executing runtime is used as framework. --- .../DependencyManager.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index 55c41f31601..228f26850f8 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -368,10 +368,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - runtimeLocation ??= Runtime.ExecutingRuntime; + if (runtimeLocation is null) + { + runtimeLocation ??= Runtime.ExecutingRuntime; + dllPaths.Add(new AssemblyPath(runtimeLocation, name => !name.StartsWith("Semmle."))); + } + else + { + dllPaths.Add(runtimeLocation); + } logger.LogInfo($".NET runtime location selected: {runtimeLocation}"); - dllPaths.Add(runtimeLocation); frameworkLocations.Add(runtimeLocation); } From 2bea927d43d2afbc5d5945abd1161e272992774d Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Fri, 5 Apr 2024 14:56:20 +0200 Subject: [PATCH 82/97] C#: Update expected test output. --- .../Assemblies.expected | 9 --------- 1 file changed, 9 deletions(-) diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.expected b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.expected index f4f8290eb75..772f640789e 100644 --- a/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.expected +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.expected @@ -17,15 +17,6 @@ | [...]/Microsoft.Win32.SystemEvents.dll | | [...]/Mono.Posix.NETStandard.dll | | [...]/Newtonsoft.Json.dll | -| [...]/Semmle.Autobuild.CSharp.dll | -| [...]/Semmle.Autobuild.Shared.dll | -| [...]/Semmle.Extraction.CSharp.DependencyFetching.dll | -| [...]/Semmle.Extraction.CSharp.Driver.dll | -| [...]/Semmle.Extraction.CSharp.Standalone.dll | -| [...]/Semmle.Extraction.CSharp.Util.dll | -| [...]/Semmle.Extraction.CSharp.dll | -| [...]/Semmle.Extraction.dll | -| [...]/Semmle.Util.dll | | [...]/System.AppContext.dll | | [...]/System.Buffers.dll | | [...]/System.Collections.Concurrent.dll | From 9eb13833faeac9c2bfdb2eaf247f1ee61907d7ff Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Tue, 9 Apr 2024 11:57:25 +0200 Subject: [PATCH 83/97] C#: Code quality improvements. --- .../AssemblyCache.cs | 4 +-- ...emblyPath.cs => AssemblyLookupLocation.cs} | 26 +++++++++---------- .../DependencyManager.Nuget.cs | 8 +++--- .../DependencyManager.cs | 20 +++++++------- 4 files changed, 29 insertions(+), 29 deletions(-) rename csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/{AssemblyPath.cs => AssemblyLookupLocation.cs} (67%) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs index f1ee65319ab..62a053870e9 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs @@ -19,12 +19,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// assembly cache. /// /// Callback for progress. - public AssemblyCache(IEnumerable paths, IEnumerable frameworkPaths, ILogger logger) + public AssemblyCache(IEnumerable paths, IEnumerable frameworkPaths, ILogger logger) { this.logger = logger; foreach (var path in paths) { - path.Process(dllsToIndex, logger); + dllsToIndex.AddRange(path.GetDlls(logger)); } IndexReferences(frameworkPaths); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyPath.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs similarity index 67% rename from csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyPath.cs rename to csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs index 816e0fef6a7..380c60cab56 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyPath.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs @@ -9,13 +9,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// Used to represent a path to an assembly or a directory containing assemblies /// and a selector function to determine which files to include, when indexing the assemblies. /// - internal sealed class AssemblyPath(string p, Func includeFileName) + internal sealed class AssemblyLookupLocation(string p, Func includeFileName) { public string Path => p; - public AssemblyPath(string p) : this(p, _ => true) { } + public AssemblyLookupLocation(string p) : this(p, _ => true) { } - public static implicit operator AssemblyPath(string path) => new(path); + public static implicit operator AssemblyLookupLocation(string path) => new(path); /// /// Finds all assemblies nested within the directory `path` @@ -33,19 +33,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } else { - logger.LogInfo($"AssemblyPath: Skipping {dll.FullName}."); + logger.LogInfo($"AssemblyLookupLocation: Skipping {dll.FullName}."); } } } /// - /// Finds all assemblies in `p` that should be indexed and adds them to - /// the list of assembly names to index. + /// Returns a list of paths to all assemblies in `p` that should be indexed. /// - /// List of assembly names to index. /// Logger - public void Process(List dllsToIndex, ILogger logger) + public List GetDlls(ILogger logger) { + var dllsToIndex = new List(); if (File.Exists(p)) { if (includeFileName(System.IO.Path.GetFileName(p))) @@ -54,24 +53,25 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } else { - logger.LogInfo($"AssemblyPath: Skipping {p}."); + logger.LogInfo($"AssemblyLookupLocation: Skipping {p}."); } - return; + return dllsToIndex; } if (Directory.Exists(p)) { - logger.LogInfo($"AssemblyPath: Finding reference DLLs in {p}..."); + logger.LogInfo($"AssemblyLookupLocation: Finding reference DLLs in {p}..."); AddReferenceDirectory(dllsToIndex, logger); } else { - logger.LogInfo("AssemblyCache: Path not found: " + p); + logger.LogInfo("AssemblyLookupLocation: Path not found: " + p); } + return dllsToIndex; } public override bool Equals(object? obj) => - obj is AssemblyPath ap && p.Equals(ap.Path); + obj is AssemblyLookupLocation ap && p.Equals(ap.Path); public override int GetHashCode() => p.GetHashCode(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs index e808235e54d..40f240f3145 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs @@ -12,7 +12,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { public sealed partial class DependencyManager { - private void RestoreNugetPackages(List allNonBinaryFiles, IEnumerable allProjects, IEnumerable allSolutions, HashSet dllPaths) + private void RestoreNugetPackages(List allNonBinaryFiles, IEnumerable allProjects, IEnumerable allSolutions, HashSet dllPaths) { try { @@ -55,7 +55,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } nugetPackageDllPaths.ExceptWith(excludedPaths); - dllPaths.UnionWith(nugetPackageDllPaths.Select(p => new AssemblyPath(p))); + dllPaths.UnionWith(nugetPackageDllPaths.Select(p => new AssemblyLookupLocation(p))); } catch (Exception exc) { @@ -72,7 +72,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching .Paths .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d)) .ToList(); - dllPaths.UnionWith(paths.Select(p => new AssemblyPath(p))); + dllPaths.UnionWith(paths.Select(p => new AssemblyLookupLocation(p))); LogAllUnusedPackages(dependencies); DownloadMissingPackages(allNonBinaryFiles, dllPaths); @@ -148,7 +148,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString())); } - private void DownloadMissingPackages(List allFiles, ISet dllPaths, bool withNugetConfig = true) + private void DownloadMissingPackages(List allFiles, ISet dllPaths, bool withNugetConfig = true) { var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo); var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index 228f26850f8..a59312a1e35 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -91,7 +91,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching this.generatedSources = new(); var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj").ToList(); var allSolutions = allNonBinaryFiles.SelectFileNamesByExtension(".sln").ToList(); - var dllPaths = allFiles.SelectFileNamesByExtension(".dll").Select(x => x).ToHashSet(); + var dllPaths = allFiles.SelectFileNamesByExtension(".dll").Select(x => new AssemblyLookupLocation(x)).ToHashSet(); logger.LogInfo($"Found {allFiles.Count} files, {nonGeneratedSources.Count} source files, {allProjects.Count} project files, {allSolutions.Count} solution files, {dllPaths.Count} DLLs."); @@ -192,7 +192,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching ]); } - private HashSet AddFrameworkDlls(HashSet dllPaths) + private HashSet AddFrameworkDlls(HashSet dllPaths) { var frameworkLocations = new HashSet(); @@ -230,7 +230,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching continue; } - dllPaths.UnionWith(dlls.Select(x => x)); + dllPaths.UnionWith(dlls.Select(x => new AssemblyLookupLocation(x))); frameworkLocations.UnionWith(dlls); } catch (Exception e) @@ -284,7 +284,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet dllPaths, ISet frameworkLocations) + private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet dllPaths, ISet frameworkLocations) { var versionFolders = GetPackageVersionSubDirectories(frameworkPath); if (versionFolders.Length > 1) @@ -313,7 +313,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching .ToArray(); } - private void RemoveFrameworkNugetPackages(ISet dllPaths, int fromIndex = 0) + private void RemoveFrameworkNugetPackages(ISet dllPaths, int fromIndex = 0) { var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks; for (var i = fromIndex; i < packagesInPrioOrder.Length; i++) @@ -322,7 +322,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - private void AddNetFrameworkDlls(ISet dllPaths, ISet frameworkLocations) + private void AddNetFrameworkDlls(ISet dllPaths, ISet frameworkLocations) { // Multiple dotnet framework packages could be present. // The order of the packages is important, we're adding the first one that is present in the nuget cache. @@ -371,7 +371,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (runtimeLocation is null) { runtimeLocation ??= Runtime.ExecutingRuntime; - dllPaths.Add(new AssemblyPath(runtimeLocation, name => !name.StartsWith("Semmle."))); + dllPaths.Add(new AssemblyLookupLocation(runtimeLocation, name => !name.StartsWith("Semmle."))); } else { @@ -382,7 +382,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching frameworkLocations.Add(runtimeLocation); } - private void RemoveNugetPackageReference(string packagePrefix, ISet dllPaths) + private void RemoveNugetPackageReference(string packagePrefix, ISet dllPaths) { var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant(); if (packageFolder == null) @@ -404,7 +404,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return fileContent.IsNewProjectStructureUsed && fileContent.UseAspNetCoreDlls; } - private void AddAspNetCoreFrameworkDlls(ISet dllPaths, ISet frameworkLocations) + private void AddAspNetCoreFrameworkDlls(ISet dllPaths, ISet frameworkLocations) { if (!IsAspNetCoreDetected()) { @@ -426,7 +426,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - private void AddMicrosoftWindowsDesktopDlls(ISet dllPaths, ISet frameworkLocations) + private void AddMicrosoftWindowsDesktopDlls(ISet dllPaths, ISet frameworkLocations) { if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework, packageDirectory) is string windowsDesktopApp) { From 99f0ed26e9791f1bdf2e2e0ea8736e2958f0a73f Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Tue, 9 Apr 2024 13:01:20 +0200 Subject: [PATCH 84/97] C#: Make the assembly lookup case insensitive on the dll file extension and log if no dlls are found in a directory. --- .../AssemblyLookupLocation.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs index 380c60cab56..8711b7f2a6c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using Semmle.Util.Logging; namespace Semmle.Extraction.CSharp.DependencyFetching @@ -25,7 +26,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// The directory to index. private void AddReferenceDirectory(List dllsToIndex, ILogger logger) { - foreach (var dll in new DirectoryInfo(p).EnumerateFiles("*.dll", SearchOption.AllDirectories)) + var dlls = new DirectoryInfo(p).EnumerateFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true, MatchCasing = MatchCasing.CaseInsensitive, AttributesToSkip = FileAttributes.None }); + if (!dlls.Any()) + { + logger.LogWarning($"AssemblyLookupLocation: No DLLs found in the path '{p}'."); + return; + } + foreach (var dll in dlls) { if (includeFileName(dll.Name)) { From a18a4fb62e1a8fc503279a87d9a3f37c03519737 Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Wed, 10 Apr 2024 11:38:27 +0100 Subject: [PATCH 85/97] Avoid magic in `TSynthLocation` definition This improves performance, because in this case magic is not beneficial. --- go/ql/lib/semmle/go/internal/Locations.qll | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/go/ql/lib/semmle/go/internal/Locations.qll b/go/ql/lib/semmle/go/internal/Locations.qll index 498ac9d1170..8b0a206b883 100644 --- a/go/ql/lib/semmle/go/internal/Locations.qll +++ b/go/ql/lib/semmle/go/internal/Locations.qll @@ -11,12 +11,19 @@ newtype TLocation = TSynthLocation(string filepath, int startline, int startcolumn, int endline, int endcolumn) { any(DataFlow::Node n).hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn) and // avoid overlap with existing DB locations - not exists(File f | - locations_default(_, f, startline, startcolumn, endline, endcolumn) and - f.getAbsolutePath() = filepath - ) + not existingDBLocation(filepath, startline, startcolumn, endline, endcolumn) } +pragma[nomagic] +private predicate existingDBLocation( + string filepath, int startline, int startcolumn, int endline, int endcolumn +) { + exists(File f | + locations_default(_, f, startline, startcolumn, endline, endcolumn) and + f.getAbsolutePath() = filepath + ) +} + /** * A location as given by a file, a start line, a start column, * an end line, and an end column. From 3b42dc25a144c8dc64fecbd01075a5b9e399dae0 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Tue, 9 Apr 2024 16:16:21 +0200 Subject: [PATCH 86/97] C#: Also use AssemblyLookupLocation for framework dlls. --- .../AssemblyLookupLocation.cs | 35 +++++++++++-------- .../DependencyManager.cs | 25 ++----------- 2 files changed, 23 insertions(+), 37 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs index 8711b7f2a6c..533b547e45d 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs @@ -10,7 +10,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// Used to represent a path to an assembly or a directory containing assemblies /// and a selector function to determine which files to include, when indexing the assemblies. /// - internal sealed class AssemblyLookupLocation(string p, Func includeFileName) + internal sealed class AssemblyLookupLocation(string p, Func includeFileName, bool indexSubdirectories = true) { public string Path => p; @@ -26,22 +26,29 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// The directory to index. private void AddReferenceDirectory(List dllsToIndex, ILogger logger) { - var dlls = new DirectoryInfo(p).EnumerateFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true, MatchCasing = MatchCasing.CaseInsensitive, AttributesToSkip = FileAttributes.None }); - if (!dlls.Any()) + try { - logger.LogWarning($"AssemblyLookupLocation: No DLLs found in the path '{p}'."); - return; + var dlls = new DirectoryInfo(p).EnumerateFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = indexSubdirectories, MatchCasing = MatchCasing.CaseInsensitive, AttributesToSkip = FileAttributes.None }); + if (!dlls.Any()) + { + logger.LogWarning($"AssemblyLookupLocation: No DLLs found in the path '{p}'."); + return; + } + foreach (var dll in dlls) + { + if (includeFileName(dll.Name)) + { + dllsToIndex.Add(dll.FullName); + } + else + { + logger.LogInfo($"AssemblyLookupLocation: Skipping {dll.FullName}."); + } + } } - foreach (var dll in dlls) + catch (Exception e) { - if (includeFileName(dll.Name)) - { - dllsToIndex.Add(dll.FullName); - } - else - { - logger.LogInfo($"AssemblyLookupLocation: Skipping {dll.FullName}."); - } + logger.LogError($"AssemblyLookupLocation: Error while searching for DLLs in '{p}': {e.Message}"); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index a59312a1e35..8115b0e8ae5 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -214,29 +214,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching continue; } - if (useSubfolders) - { - dllPaths.Add(path); - frameworkLocations.Add(path); - continue; - } - - try - { - var dlls = Directory.GetFiles(path, "*.dll", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive }); - if (dlls.Length == 0) - { - logger.LogError($"No DLLs found in specified framework reference path '{path}'."); - continue; - } - - dllPaths.UnionWith(dlls.Select(x => new AssemblyLookupLocation(x))); - frameworkLocations.UnionWith(dlls); - } - catch (Exception e) - { - logger.LogError($"Error while searching for DLLs in '{path}': {e.Message}"); - } + dllPaths.Add(new AssemblyLookupLocation(path, _ => true, useSubfolders)); + frameworkLocations.Add(path); } return frameworkLocations; From 78ca691912425c515cc6fccce1af49efd8d7b07a Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 10 Apr 2024 13:12:17 +0200 Subject: [PATCH 87/97] Python: remove deprecated points-to test for zope --- .../ql/test/library-tests/web/zope/Test.expected | 3 --- python/ql/test/library-tests/web/zope/Test.ql | 10 ---------- python/ql/test/library-tests/web/zope/options | 1 - python/ql/test/library-tests/web/zope/test.py | 14 -------------- 4 files changed, 28 deletions(-) delete mode 100644 python/ql/test/library-tests/web/zope/Test.expected delete mode 100644 python/ql/test/library-tests/web/zope/Test.ql delete mode 100644 python/ql/test/library-tests/web/zope/options delete mode 100644 python/ql/test/library-tests/web/zope/test.py diff --git a/python/ql/test/library-tests/web/zope/Test.expected b/python/ql/test/library-tests/web/zope/Test.expected deleted file mode 100644 index bb3ca1f441e..00000000000 --- a/python/ql/test/library-tests/web/zope/Test.expected +++ /dev/null @@ -1,3 +0,0 @@ -| 12 | ControlFlowNode for implementer | class implementer | ../../../query-tests/Security/lib/zope/interface/__init__.py:5 | -| 13 | ControlFlowNode for IThing | class IThing | test.py:4 | -| 14 | ControlFlowNode for Thing | class Thing | test.py:9 | diff --git a/python/ql/test/library-tests/web/zope/Test.ql b/python/ql/test/library-tests/web/zope/Test.ql deleted file mode 100644 index ca705ac292e..00000000000 --- a/python/ql/test/library-tests/web/zope/Test.ql +++ /dev/null @@ -1,10 +0,0 @@ -import python -import semmle.python.TestUtils - -from ControlFlowNode f, Value v, ControlFlowNode x -where - exists(ExprStmt s | s.getValue().getAFlowNode() = f) and - f.pointsTo(v, x) and - f.getLocation().getFile().getBaseName() = "test.py" -select f.getLocation().getStartLine(), f.toString(), v.toString(), - remove_library_prefix(x.getLocation()) diff --git a/python/ql/test/library-tests/web/zope/options b/python/ql/test/library-tests/web/zope/options deleted file mode 100644 index 7fb713d5924..00000000000 --- a/python/ql/test/library-tests/web/zope/options +++ /dev/null @@ -1 +0,0 @@ -semmle-extractor-options: --max-import-depth=3 -p ../../../query-tests/Security/lib/ diff --git a/python/ql/test/library-tests/web/zope/test.py b/python/ql/test/library-tests/web/zope/test.py deleted file mode 100644 index 64ac9138d7e..00000000000 --- a/python/ql/test/library-tests/web/zope/test.py +++ /dev/null @@ -1,14 +0,0 @@ - -from zope.interface import Interface, implementer - -class IThing(Interface): - pass - - -@implementer(IThing) -class Thing(object): - pass - -implementer -IThing -Thing From 9615e2ded98443b05a7166687171e6c33fa14f66 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 10 Apr 2024 13:12:36 +0200 Subject: [PATCH 88/97] Python: Remove deprecated stubs for points-to tests I grep'ed through all our options files, and couldn't find any tests that relies on these anymore :+1: --- .../Security/lib/Crypto/Cipher/AES.py | 3 - .../Security/lib/Crypto/Cipher/ARC4.py | 3 - .../Security/lib/Crypto/Cipher/__init__.py | 1 - .../Security/lib/Crypto/PublicKey/DSA.py | 2 - .../Security/lib/Crypto/PublicKey/RSA.py | 2 - .../Security/lib/Crypto/PublicKey/__init__.py | 0 .../Security/lib/Crypto/__init__.py | 0 .../test/query-tests/Security/lib/airspeed.py | 3 - .../test/query-tests/Security/lib/base64.py | 2 - .../test/query-tests/Security/lib/bottle.py | 69 ------------------- .../Security/lib/cheetah/Template/__init__.py | 2 - .../Security/lib/cherrypy/__init__.py | 15 ---- .../Security/lib/cherrypy/_helper.py | 31 --------- .../test/query-tests/Security/lib/chevron.py | 6 -- .../Security/lib/cryptography/__init__.py | 0 .../lib/cryptography/hazmat/__init__.py | 0 .../hazmat/primitives/__init__.py | 0 .../hazmat/primitives/asymmetric/__init__.py | 0 .../hazmat/primitives/asymmetric/dsa.py | 4 -- .../hazmat/primitives/asymmetric/ec.py | 22 ------ .../hazmat/primitives/asymmetric/rsa.py | 3 - .../hazmat/primitives/ciphers/__init__.py | 3 - .../hazmat/primitives/ciphers/algorithms.py | 3 - .../ql/test/query-tests/Security/lib/dill.py | 2 - .../Security/lib/django/__init__.py | 0 .../Security/lib/django/conf/__init__.py | 0 .../Security/lib/django/conf/urls.py | 7 -- .../Security/lib/django/db/__init__.py | 1 - .../Security/lib/django/db/models/__init__.py | 2 - .../lib/django/db/models/expressions.py | 2 - .../Security/lib/django/http/__init__.py | 2 - .../Security/lib/django/http/request.py | 2 - .../Security/lib/django/http/response.py | 46 ------------- .../Security/lib/django/shortcuts.py | 8 --- .../query-tests/Security/lib/django/urls.py | 7 -- .../Security/lib/django/views/__init__.py | 3 - .../Security/lib/django/views/generic.py | 3 - .../Security/lib/fabric/__init__.py | 3 - .../query-tests/Security/lib/fabric/api.py | 29 -------- .../Security/lib/fabric/connection.py | 15 ---- .../query-tests/Security/lib/fabric/group.py | 11 --- .../query-tests/Security/lib/fabric/tasks.py | 2 - .../Security/lib/falcon/__init__.py | 4 -- .../query-tests/Security/lib/falcon/api.py | 14 ---- .../Security/lib/falcon/request.py | 3 - .../Security/lib/falcon/response.py | 4 -- .../Security/lib/flask/__init__.py | 33 --------- .../query-tests/Security/lib/flask/globals.py | 6 -- .../query-tests/Security/lib/flask/urls.py | 0 .../query-tests/Security/lib/flask/views.py | 6 -- .../Security/lib/invoke/__init__.py | 8 --- .../Security/lib/invoke/context.py | 3 - .../query-tests/Security/lib/invoke/tasks.py | 2 - .../test/query-tests/Security/lib/jinja2.py | 23 ------- .../Security/lib/libxml2/__init__.py | 10 --- .../query-tests/Security/lib/lxml/__init__.py | 0 .../query-tests/Security/lib/lxml/etree.py | 37 ---------- .../test/query-tests/Security/lib/marshall.py | 2 - .../query-tests/Security/lib/os/__init__.py | 17 ----- .../test/query-tests/Security/lib/os/path.py | 5 -- .../Security/lib/paramiko/__init__.py | 0 .../Security/lib/paramiko/client.py | 15 ---- .../test/query-tests/Security/lib/pickle.py | 2 - .../query-tests/Security/lib/pyOpenSSL/SSL.py | 10 --- .../Security/lib/pyOpenSSL/__init__.py | 0 .../Security/lib/pyramid/__init__.py | 0 .../Security/lib/pyramid/response.py | 2 - .../query-tests/Security/lib/pyramid/view.py | 7 -- .../test/query-tests/Security/lib/requests.py | 21 ------ .../query-tests/Security/lib/subprocess.py | 2 - python/ql/test/query-tests/Security/lib/tg.py | 30 -------- .../Security/lib/tornado/__init__.py | 0 .../query-tests/Security/lib/tornado/web.py | 2 - .../query-tests/Security/lib/traceback.py | 2 - .../Security/lib/twisted/__init__.py | 0 .../Security/lib/twisted/web/__init__.py | 0 .../Security/lib/twisted/web/http.py | 2 - .../Security/lib/twisted/web/resource.py | 2 - .../ql/test/query-tests/Security/lib/yaml.py | 2 - .../query-tests/Security/lib/zope/__init__.py | 0 .../Security/lib/zope/interface/__init__.py | 9 --- 81 files changed, 604 deletions(-) delete mode 100644 python/ql/test/query-tests/Security/lib/Crypto/Cipher/AES.py delete mode 100644 python/ql/test/query-tests/Security/lib/Crypto/Cipher/ARC4.py delete mode 100644 python/ql/test/query-tests/Security/lib/Crypto/Cipher/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/Crypto/PublicKey/DSA.py delete mode 100644 python/ql/test/query-tests/Security/lib/Crypto/PublicKey/RSA.py delete mode 100644 python/ql/test/query-tests/Security/lib/Crypto/PublicKey/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/Crypto/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/airspeed.py delete mode 100644 python/ql/test/query-tests/Security/lib/base64.py delete mode 100644 python/ql/test/query-tests/Security/lib/bottle.py delete mode 100644 python/ql/test/query-tests/Security/lib/cheetah/Template/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/cherrypy/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/cherrypy/_helper.py delete mode 100644 python/ql/test/query-tests/Security/lib/chevron.py delete mode 100644 python/ql/test/query-tests/Security/lib/cryptography/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/cryptography/hazmat/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/dsa.py delete mode 100644 python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/ec.py delete mode 100644 python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/rsa.py delete mode 100644 python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/ciphers/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/ciphers/algorithms.py delete mode 100644 python/ql/test/query-tests/Security/lib/dill.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/conf/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/conf/urls.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/db/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/db/models/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/db/models/expressions.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/http/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/http/request.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/http/response.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/shortcuts.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/urls.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/views/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/views/generic.py delete mode 100644 python/ql/test/query-tests/Security/lib/fabric/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/fabric/api.py delete mode 100644 python/ql/test/query-tests/Security/lib/fabric/connection.py delete mode 100644 python/ql/test/query-tests/Security/lib/fabric/group.py delete mode 100644 python/ql/test/query-tests/Security/lib/fabric/tasks.py delete mode 100644 python/ql/test/query-tests/Security/lib/falcon/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/falcon/api.py delete mode 100644 python/ql/test/query-tests/Security/lib/falcon/request.py delete mode 100644 python/ql/test/query-tests/Security/lib/falcon/response.py delete mode 100644 python/ql/test/query-tests/Security/lib/flask/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/flask/globals.py delete mode 100644 python/ql/test/query-tests/Security/lib/flask/urls.py delete mode 100644 python/ql/test/query-tests/Security/lib/flask/views.py delete mode 100644 python/ql/test/query-tests/Security/lib/invoke/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/invoke/context.py delete mode 100644 python/ql/test/query-tests/Security/lib/invoke/tasks.py delete mode 100644 python/ql/test/query-tests/Security/lib/jinja2.py delete mode 100644 python/ql/test/query-tests/Security/lib/libxml2/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/lxml/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/lxml/etree.py delete mode 100644 python/ql/test/query-tests/Security/lib/marshall.py delete mode 100644 python/ql/test/query-tests/Security/lib/os/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/os/path.py delete mode 100644 python/ql/test/query-tests/Security/lib/paramiko/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/paramiko/client.py delete mode 100644 python/ql/test/query-tests/Security/lib/pickle.py delete mode 100644 python/ql/test/query-tests/Security/lib/pyOpenSSL/SSL.py delete mode 100644 python/ql/test/query-tests/Security/lib/pyOpenSSL/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/pyramid/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/pyramid/response.py delete mode 100644 python/ql/test/query-tests/Security/lib/pyramid/view.py delete mode 100644 python/ql/test/query-tests/Security/lib/requests.py delete mode 100644 python/ql/test/query-tests/Security/lib/subprocess.py delete mode 100644 python/ql/test/query-tests/Security/lib/tg.py delete mode 100644 python/ql/test/query-tests/Security/lib/tornado/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/tornado/web.py delete mode 100644 python/ql/test/query-tests/Security/lib/traceback.py delete mode 100644 python/ql/test/query-tests/Security/lib/twisted/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/twisted/web/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/twisted/web/http.py delete mode 100644 python/ql/test/query-tests/Security/lib/twisted/web/resource.py delete mode 100644 python/ql/test/query-tests/Security/lib/yaml.py delete mode 100644 python/ql/test/query-tests/Security/lib/zope/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/zope/interface/__init__.py diff --git a/python/ql/test/query-tests/Security/lib/Crypto/Cipher/AES.py b/python/ql/test/query-tests/Security/lib/Crypto/Cipher/AES.py deleted file mode 100644 index bcab58405ea..00000000000 --- a/python/ql/test/query-tests/Security/lib/Crypto/Cipher/AES.py +++ /dev/null @@ -1,3 +0,0 @@ - -def new(*args): - pass diff --git a/python/ql/test/query-tests/Security/lib/Crypto/Cipher/ARC4.py b/python/ql/test/query-tests/Security/lib/Crypto/Cipher/ARC4.py deleted file mode 100644 index bcab58405ea..00000000000 --- a/python/ql/test/query-tests/Security/lib/Crypto/Cipher/ARC4.py +++ /dev/null @@ -1,3 +0,0 @@ - -def new(*args): - pass diff --git a/python/ql/test/query-tests/Security/lib/Crypto/Cipher/__init__.py b/python/ql/test/query-tests/Security/lib/Crypto/Cipher/__init__.py deleted file mode 100644 index ab22f65be6e..00000000000 --- a/python/ql/test/query-tests/Security/lib/Crypto/Cipher/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__ = ['AES', 'ARC4'] diff --git a/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/DSA.py b/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/DSA.py deleted file mode 100644 index e002a77697e..00000000000 --- a/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/DSA.py +++ /dev/null @@ -1,2 +0,0 @@ -def generate(bits): - pass diff --git a/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/RSA.py b/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/RSA.py deleted file mode 100644 index e002a77697e..00000000000 --- a/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/RSA.py +++ /dev/null @@ -1,2 +0,0 @@ -def generate(bits): - pass diff --git a/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/__init__.py b/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/Crypto/__init__.py b/python/ql/test/query-tests/Security/lib/Crypto/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/airspeed.py b/python/ql/test/query-tests/Security/lib/airspeed.py deleted file mode 100644 index aea81c72b71..00000000000 --- a/python/ql/test/query-tests/Security/lib/airspeed.py +++ /dev/null @@ -1,3 +0,0 @@ -class Template: - def __init__(self, content, filename=""): - pass diff --git a/python/ql/test/query-tests/Security/lib/base64.py b/python/ql/test/query-tests/Security/lib/base64.py deleted file mode 100644 index a2b97ca63ac..00000000000 --- a/python/ql/test/query-tests/Security/lib/base64.py +++ /dev/null @@ -1,2 +0,0 @@ -def decodestring(s): - return None diff --git a/python/ql/test/query-tests/Security/lib/bottle.py b/python/ql/test/query-tests/Security/lib/bottle.py deleted file mode 100644 index fe42507852c..00000000000 --- a/python/ql/test/query-tests/Security/lib/bottle.py +++ /dev/null @@ -1,69 +0,0 @@ - -class Bottle(object): - - def route(self, path=None, method='GET', **options): - pass - - def get(self, path=None, method='GET', **options): - """ Equals :meth:`route`. """ - return self.route(path, method, **options) - - def post(self, path=None, method='POST', **options): - """ Equals :meth:`route` with a ``POST`` method parameter. """ - return self.route(path, method, **options) - - def put(self, path=None, method='PUT', **options): - """ Equals :meth:`route` with a ``PUT`` method parameter. """ - return self.route(path, method, **options) - - def delete(self, path=None, method='DELETE', **options): - """ Equals :meth:`route` with a ``DELETE`` method parameter. """ - return self.route(path, method, **options) - - def error(self, code=500): - """ Decorator: Register an output handler for a HTTP error code""" - def wrapper(handler): - self.error_handler[int(code)] = handler - return handler - return wrapper - -#Use same wrapper logic as the original `bottle` code. - -def make_default_app_wrapper(name): - """ Return a callable that relays calls to the current default app. """ - - @functools.wraps(getattr(Bottle, name)) - def wrapper(*a, **ka): - return getattr(app(), name)(*a, **ka) - - return wrapper - -route = make_default_app_wrapper('route') -get = make_default_app_wrapper('get') -post = make_default_app_wrapper('post') -put = make_default_app_wrapper('put') -delete = make_default_app_wrapper('delete') -patch = make_default_app_wrapper('patch') -error = make_default_app_wrapper('error') -mount = make_default_app_wrapper('mount') -hook = make_default_app_wrapper('hook') -install = make_default_app_wrapper('install') -uninstall = make_default_app_wrapper('uninstall') -url = make_default_app_wrapper('get_url') - -class LocalProxy(object): - pass - -class LocalRequest(LocalProxy): - pass - -class LocalResponse(LocalProxy): - pass - - -request = LocalRequest() -response = LocalResponse() - - -def redirect(url, code=None): - pass diff --git a/python/ql/test/query-tests/Security/lib/cheetah/Template/__init__.py b/python/ql/test/query-tests/Security/lib/cheetah/Template/__init__.py deleted file mode 100644 index 04b862eafea..00000000000 --- a/python/ql/test/query-tests/Security/lib/cheetah/Template/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -class Template(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/cherrypy/__init__.py b/python/ql/test/query-tests/Security/lib/cherrypy/__init__.py deleted file mode 100644 index a1ae780e3b5..00000000000 --- a/python/ql/test/query-tests/Security/lib/cherrypy/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ - - -from ._helper import expose, popargs, url - -class _ThreadLocalProxy(object): - def __getattr__(self, name): - pass - - -request = _ThreadLocalProxy('request') -response = _ThreadLocalProxy('response') - -def quickstart(root=None, script_name='', config=None): - """Mount the given root, start the builtin server (and engine), then block.""" - pass diff --git a/python/ql/test/query-tests/Security/lib/cherrypy/_helper.py b/python/ql/test/query-tests/Security/lib/cherrypy/_helper.py deleted file mode 100644 index fc3f3604a57..00000000000 --- a/python/ql/test/query-tests/Security/lib/cherrypy/_helper.py +++ /dev/null @@ -1,31 +0,0 @@ -def expose(func=None, alias=None): - """Expose the function or class. - Optionally provide an alias or set of aliases. - """ - def expose_(func): - func.exposed = True - return func - - return expose_ - - -def popargs(*args, **kwargs): - """Decorate _cp_dispatch.""" - - def decorated(cls_or_self=None, vpath=None): - if inspect.isclass(cls_or_self): - # cherrypy.popargs is a class decorator - return cls - - # We're in the actual function - self = cls_or_self - if vpath: - return getattr(self, vpath.pop(0), None) - else: - return self - - return decorated - -def url(path='', qs='', script_name=None, base=None, relative=None): - #Do some opaque stuff here... - return new_url diff --git a/python/ql/test/query-tests/Security/lib/chevron.py b/python/ql/test/query-tests/Security/lib/chevron.py deleted file mode 100644 index 012ba58fdb8..00000000000 --- a/python/ql/test/query-tests/Security/lib/chevron.py +++ /dev/null @@ -1,6 +0,0 @@ - - -def render(template='', data={}, partials_path='.', partials_ext='mustache', - partials_dict={}, padding='', def_ldel='{{', def_rdel='}}', - scopes=None): - pass diff --git a/python/ql/test/query-tests/Security/lib/cryptography/__init__.py b/python/ql/test/query-tests/Security/lib/cryptography/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/__init__.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/__init__.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/__init__.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/dsa.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/dsa.py deleted file mode 100644 index b4c37f6b540..00000000000 --- a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/dsa.py +++ /dev/null @@ -1,4 +0,0 @@ - -def generate_private_key(key_size, backend): - pass - diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/ec.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/ec.py deleted file mode 100644 index d2eb4b0f1af..00000000000 --- a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/ec.py +++ /dev/null @@ -1,22 +0,0 @@ - -def generate_private_key(curve, backend): - pass - -class SECT571R1(object): - name = "sect571r1" - key_size = 570 - - -class SECP384R1(object): - name = "secp384r1" - key_size = 384 - - -class SECP224R1(object): - name = "secp224r1" - key_size = 224 - - -class SECT163K1(object): - name = "sect163k1" - key_size = 163 \ No newline at end of file diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/rsa.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/rsa.py deleted file mode 100644 index 3a5c91734c2..00000000000 --- a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/rsa.py +++ /dev/null @@ -1,3 +0,0 @@ - -def generate_private_key(public_exponent, key_size, backend): - pass diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/ciphers/__init__.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/ciphers/__init__.py deleted file mode 100644 index f37f14e88f8..00000000000 --- a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/ciphers/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ - -class Cipher(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/ciphers/algorithms.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/ciphers/algorithms.py deleted file mode 100644 index e423804e4f5..00000000000 --- a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/ciphers/algorithms.py +++ /dev/null @@ -1,3 +0,0 @@ - -class ARC4(object): - name = "RC4" diff --git a/python/ql/test/query-tests/Security/lib/dill.py b/python/ql/test/query-tests/Security/lib/dill.py deleted file mode 100644 index 410fa21087e..00000000000 --- a/python/ql/test/query-tests/Security/lib/dill.py +++ /dev/null @@ -1,2 +0,0 @@ -def loads(*args, **kwargs): - return None diff --git a/python/ql/test/query-tests/Security/lib/django/__init__.py b/python/ql/test/query-tests/Security/lib/django/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/django/conf/__init__.py b/python/ql/test/query-tests/Security/lib/django/conf/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/django/conf/urls.py b/python/ql/test/query-tests/Security/lib/django/conf/urls.py deleted file mode 100644 index 49a248e16aa..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/conf/urls.py +++ /dev/null @@ -1,7 +0,0 @@ -# https://docs.djangoproject.com/en/1.11/_modules/django/conf/urls/#url -def url(regex, view, kwargs=None, name=None): - pass - - -def patterns(*urls): - pass diff --git a/python/ql/test/query-tests/Security/lib/django/db/__init__.py b/python/ql/test/query-tests/Security/lib/django/db/__init__.py deleted file mode 100644 index 3e291c5731d..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/db/__init__.py +++ /dev/null @@ -1 +0,0 @@ -connection = object() diff --git a/python/ql/test/query-tests/Security/lib/django/db/models/__init__.py b/python/ql/test/query-tests/Security/lib/django/db/models/__init__.py deleted file mode 100644 index eb9c72adc45..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/db/models/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -class Model: - pass diff --git a/python/ql/test/query-tests/Security/lib/django/db/models/expressions.py b/python/ql/test/query-tests/Security/lib/django/db/models/expressions.py deleted file mode 100644 index d7e0d1c27b6..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/db/models/expressions.py +++ /dev/null @@ -1,2 +0,0 @@ -class RawSQL: - pass diff --git a/python/ql/test/query-tests/Security/lib/django/http/__init__.py b/python/ql/test/query-tests/Security/lib/django/http/__init__.py deleted file mode 100644 index f2ac6d2c55b..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/http/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .response import * -from .request import HttpRequest diff --git a/python/ql/test/query-tests/Security/lib/django/http/request.py b/python/ql/test/query-tests/Security/lib/django/http/request.py deleted file mode 100644 index 93afdaf29b7..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/http/request.py +++ /dev/null @@ -1,2 +0,0 @@ -class HttpRequest(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/django/http/response.py b/python/ql/test/query-tests/Security/lib/django/http/response.py deleted file mode 100644 index 1110a3cde19..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/http/response.py +++ /dev/null @@ -1,46 +0,0 @@ -class HttpResponseBase(object): - status_code = 200 - - def __init__(self, content_type=None, status=None, reason=None, charset=None): - pass - - -class HttpResponse(HttpResponseBase): - def __init__(self, content=b"", *args, **kwargs): - super().__init__(*args, **kwargs) - # Content is a bytestring. See the `content` property methods. - self.content = content - - -class HttpResponseRedirectBase(HttpResponse): - def __init__(self, redirect_to, *args, **kwargs): - super().__init__(*args, **kwargs) - ... - - -class HttpResponsePermanentRedirect(HttpResponseRedirectBase): - status_code = 301 - - -class HttpResponseRedirect(HttpResponseRedirectBase): - status_code = 302 - - -class HttpResponseNotFound(HttpResponse): - status_code = 404 - - -class JsonResponse(HttpResponse): - - def __init__( - self, - data, - encoder=..., - safe=True, - json_dumps_params=None, - **kwargs - ): - # fake code to represent what is going on :) - kwargs.setdefault("content_type", "application/json") - data = str(data) - super().__init__(content=data, **kwargs) diff --git a/python/ql/test/query-tests/Security/lib/django/shortcuts.py b/python/ql/test/query-tests/Security/lib/django/shortcuts.py deleted file mode 100644 index 4783729178a..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/shortcuts.py +++ /dev/null @@ -1,8 +0,0 @@ -# see https://docs.djangoproject.com/en/1.11/_modules/django/shortcuts/#redirect -# https://github.com/django/django/blob/86908785076b2bbc31b908781da6b6ad1779b18b/django/shortcuts.py - -def render(request, template_name, context=None, content_type=None, status=None, using=None): - pass - -def redirect(to, *args, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/django/urls.py b/python/ql/test/query-tests/Security/lib/django/urls.py deleted file mode 100644 index dc28eece874..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/urls.py +++ /dev/null @@ -1,7 +0,0 @@ -from functools import partial - -def _path(route, view, kwargs=None, name=None, Pattern=None): - pass - -path = partial(_path, Pattern='RoutePattern (but this is a mock)') -re_path = partial(_path, Pattern='RegexPattern (but this is a mock)') diff --git a/python/ql/test/query-tests/Security/lib/django/views/__init__.py b/python/ql/test/query-tests/Security/lib/django/views/__init__.py deleted file mode 100644 index 47de98f1351..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/views/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# For django 2.x and 3.x -class View: - pass diff --git a/python/ql/test/query-tests/Security/lib/django/views/generic.py b/python/ql/test/query-tests/Security/lib/django/views/generic.py deleted file mode 100644 index edb71bed762..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/views/generic.py +++ /dev/null @@ -1,3 +0,0 @@ -# For django 1.x -class View: - pass diff --git a/python/ql/test/query-tests/Security/lib/fabric/__init__.py b/python/ql/test/query-tests/Security/lib/fabric/__init__.py deleted file mode 100644 index fcdb406611e..00000000000 --- a/python/ql/test/query-tests/Security/lib/fabric/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .connection import Connection -from .group import Group, SerialGroup, ThreadingGroup -from .tasks import task diff --git a/python/ql/test/query-tests/Security/lib/fabric/api.py b/python/ql/test/query-tests/Security/lib/fabric/api.py deleted file mode 100644 index b26d5a04b6d..00000000000 --- a/python/ql/test/query-tests/Security/lib/fabric/api.py +++ /dev/null @@ -1,29 +0,0 @@ -# For the 1.x version - -def needs_host(func): - @wraps(func) - def inner(*args, **kwargs): - return func(*args, **kwargs) - return inner - - -def local(command, capture=False, shell=None): - pass - - -@needs_host -def run(command, shell=True, pty=True, combine_stderr=None, quiet=False, - warn_only=False, stdout=None, stderr=None, timeout=None, shell_escape=None, - capture_buffer_size=None): - pass - - -@needs_host -def sudo(command, shell=True, pty=True, combine_stderr=None, user=None, - quiet=False, warn_only=False, stdout=None, stderr=None, group=None, - timeout=None, shell_escape=None, capture_buffer_size=None): - pass - -# https://github.com/fabric/fabric/blob/1.14/fabric/tasks.py#L281 -def execute(task, *args, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/fabric/connection.py b/python/ql/test/query-tests/Security/lib/fabric/connection.py deleted file mode 100644 index e046aa4e744..00000000000 --- a/python/ql/test/query-tests/Security/lib/fabric/connection.py +++ /dev/null @@ -1,15 +0,0 @@ -from invoke import Context - -@decorator -def opens(method, self, *args, **kwargs): - self.open() - return method(self, *args, **kwargs) - -class Connection(Context): - - def open(self): - pass - - @opens - def run(self, command, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/fabric/group.py b/python/ql/test/query-tests/Security/lib/fabric/group.py deleted file mode 100644 index 131231d2268..00000000000 --- a/python/ql/test/query-tests/Security/lib/fabric/group.py +++ /dev/null @@ -1,11 +0,0 @@ -class Group(list): - def run(self, *args, **kwargs): - raise NotImplementedError - -class SerialGroup(Group): - def run(self, *args, **kwargs): - pass - -class ThreadingGroup(Group): - def run(self, *args, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/fabric/tasks.py b/python/ql/test/query-tests/Security/lib/fabric/tasks.py deleted file mode 100644 index 08968e68ab5..00000000000 --- a/python/ql/test/query-tests/Security/lib/fabric/tasks.py +++ /dev/null @@ -1,2 +0,0 @@ -def task(*args, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/falcon/__init__.py b/python/ql/test/query-tests/Security/lib/falcon/__init__.py deleted file mode 100644 index 5983e6a41a5..00000000000 --- a/python/ql/test/query-tests/Security/lib/falcon/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ - -from falcon.api import API -from falcon.request import Request -from falcon.response import Response diff --git a/python/ql/test/query-tests/Security/lib/falcon/api.py b/python/ql/test/query-tests/Security/lib/falcon/api.py deleted file mode 100644 index 2763d8ca3e1..00000000000 --- a/python/ql/test/query-tests/Security/lib/falcon/api.py +++ /dev/null @@ -1,14 +0,0 @@ - -"""Falcon API class.""" - -class API(object): - - def add_route(self, uri_template, resource, **kwargs): - pass - - def add_sink(self, sink, prefix=r'/'): - pass - - def add_error_handler(self, exception, handler=None): - pass - diff --git a/python/ql/test/query-tests/Security/lib/falcon/request.py b/python/ql/test/query-tests/Security/lib/falcon/request.py deleted file mode 100644 index 4e39d52d841..00000000000 --- a/python/ql/test/query-tests/Security/lib/falcon/request.py +++ /dev/null @@ -1,3 +0,0 @@ - -class Request(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/falcon/response.py b/python/ql/test/query-tests/Security/lib/falcon/response.py deleted file mode 100644 index d03bbee54ac..00000000000 --- a/python/ql/test/query-tests/Security/lib/falcon/response.py +++ /dev/null @@ -1,4 +0,0 @@ - - -class Response(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/flask/__init__.py b/python/ql/test/query-tests/Security/lib/flask/__init__.py deleted file mode 100644 index 73ce61e7ccf..00000000000 --- a/python/ql/test/query-tests/Security/lib/flask/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -from .globals import request -from .globals import current_app - -class Flask(object): - # Only some methods mocked, signature copied from - # https://flask.palletsprojects.com/en/1.1.x/api/#flask.Flask - def run(host=None, port=None, debug=None, load_dotenv=True, **options): - pass - - def make_response(rv): - pass - - def add_url_rule(rule, endpoint=None, view_func=None, provide_automatic_options=None, **options): - pass - -class Response(object): - pass - -def redirect(location, code=302, Response=None): - pass - -def make_response(rv): - if not args: - return current_app.response_class() - if len(args) == 1: - args = args[0] - return current_app.make_response(args) - -def escape(txt): - return Markup.escape(txt) - -def render_template_string(source, **context): - pass \ No newline at end of file diff --git a/python/ql/test/query-tests/Security/lib/flask/globals.py b/python/ql/test/query-tests/Security/lib/flask/globals.py deleted file mode 100644 index 36320cb4886..00000000000 --- a/python/ql/test/query-tests/Security/lib/flask/globals.py +++ /dev/null @@ -1,6 +0,0 @@ - -class LocalProxy(object): - pass - -request = LocalProxy() -current_app = LocalProxy() diff --git a/python/ql/test/query-tests/Security/lib/flask/urls.py b/python/ql/test/query-tests/Security/lib/flask/urls.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/flask/views.py b/python/ql/test/query-tests/Security/lib/flask/views.py deleted file mode 100644 index 1b7007f20da..00000000000 --- a/python/ql/test/query-tests/Security/lib/flask/views.py +++ /dev/null @@ -1,6 +0,0 @@ -class View(object): - pass - - -class MethodView(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/invoke/__init__.py b/python/ql/test/query-tests/Security/lib/invoke/__init__.py deleted file mode 100644 index 1cb865d16ff..00000000000 --- a/python/ql/test/query-tests/Security/lib/invoke/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from .context import Context -from .tasks import task - -def run(command, **kwargs): - pass - -def sudo(command, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/invoke/context.py b/python/ql/test/query-tests/Security/lib/invoke/context.py deleted file mode 100644 index dff247b23f1..00000000000 --- a/python/ql/test/query-tests/Security/lib/invoke/context.py +++ /dev/null @@ -1,3 +0,0 @@ -class Context(object): - def run(self, command, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/invoke/tasks.py b/python/ql/test/query-tests/Security/lib/invoke/tasks.py deleted file mode 100644 index 08968e68ab5..00000000000 --- a/python/ql/test/query-tests/Security/lib/invoke/tasks.py +++ /dev/null @@ -1,2 +0,0 @@ -def task(*args, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/jinja2.py b/python/ql/test/query-tests/Security/lib/jinja2.py deleted file mode 100644 index 7c95c1417ab..00000000000 --- a/python/ql/test/query-tests/Security/lib/jinja2.py +++ /dev/null @@ -1,23 +0,0 @@ - -class Environment(object): - - def __init__(self, loader, autoescape): - pass - -class Template(object): - - def __init__(self, source, autoescape): - pass - -def select_autoescape(files=[]): - def autoescape(template_name): - pass - return autoescape - -class FileSystemLoader(object): - - def __init__(self, searchpath): - pass - -def from_string(source, globals=None, template_class=None): - pass \ No newline at end of file diff --git a/python/ql/test/query-tests/Security/lib/libxml2/__init__.py b/python/ql/test/query-tests/Security/lib/libxml2/__init__.py deleted file mode 100644 index 057488829f4..00000000000 --- a/python/ql/test/query-tests/Security/lib/libxml2/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -def parseFile(filename): - return xmlDoc(_obj=None) - - -class xmlDoc(Object): - def __init__(self, _obj=None): - pass - - def xpathEval(self, expr): - pass diff --git a/python/ql/test/query-tests/Security/lib/lxml/__init__.py b/python/ql/test/query-tests/Security/lib/lxml/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/lxml/etree.py b/python/ql/test/query-tests/Security/lib/lxml/etree.py deleted file mode 100644 index 139553b0d6c..00000000000 --- a/python/ql/test/query-tests/Security/lib/lxml/etree.py +++ /dev/null @@ -1,37 +0,0 @@ -class _ElementTree(object): - def xpath(self, _path, namespaces=None, extensions=None, smart_strings=True, **_variables): - pass - - def xslt(self, _xslt, extensions=None, access_control=None, **_kw): - pass - - -class ETXPath(object): - def __init__(self, path, extensions=None, regexp=True, smart_strings=True): - pass - - -class XPath(object): - def __init__(self, path, namespaces=None, extensions=None, regexp=True, smart_strings=True): - pass - - -class XSLT(object): - def __init__(self, xslt_input, extensions=None, regexp=True, access_control=None): - pass - - -def parse(self, parser=None, base_url=None): - return _ElementTree() - - -def fromstring(self, text, parser=None, base_url=None): - pass - - -def fromstringlist(self, strings, parser=None): - pass - - -def XML(self, text, parser=None, base_url=None): - pass diff --git a/python/ql/test/query-tests/Security/lib/marshall.py b/python/ql/test/query-tests/Security/lib/marshall.py deleted file mode 100644 index 410fa21087e..00000000000 --- a/python/ql/test/query-tests/Security/lib/marshall.py +++ /dev/null @@ -1,2 +0,0 @@ -def loads(*args, **kwargs): - return None diff --git a/python/ql/test/query-tests/Security/lib/os/__init__.py b/python/ql/test/query-tests/Security/lib/os/__init__.py deleted file mode 100644 index 397e2eb12da..00000000000 --- a/python/ql/test/query-tests/Security/lib/os/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -def system(cmd, *args, **kwargs): - return None - -def popen(cmd, *args, **kwargs): - return None - -def chmod(path, mode): - pass - -def open(path, flags, mode): - pass - -def tempnam(*args): - pass - -def tmpnam(*args): - pass diff --git a/python/ql/test/query-tests/Security/lib/os/path.py b/python/ql/test/query-tests/Security/lib/os/path.py deleted file mode 100644 index d54092e80b0..00000000000 --- a/python/ql/test/query-tests/Security/lib/os/path.py +++ /dev/null @@ -1,5 +0,0 @@ -def join(a, *b): - return a + "/" + "/".join(b) - -def normpath(x): - return x diff --git a/python/ql/test/query-tests/Security/lib/paramiko/__init__.py b/python/ql/test/query-tests/Security/lib/paramiko/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/paramiko/client.py b/python/ql/test/query-tests/Security/lib/paramiko/client.py deleted file mode 100644 index 6343fbe78dd..00000000000 --- a/python/ql/test/query-tests/Security/lib/paramiko/client.py +++ /dev/null @@ -1,15 +0,0 @@ -class SSHClient(object): - def __init__(self, *args, **kwargs): - pass - - def set_missing_host_key_policy(self, *args, **kwargs): - pass - -class AutoAddPolicy(object): - pass - -class WarningPolicy(object): - pass - -class RejectPolicy(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/pickle.py b/python/ql/test/query-tests/Security/lib/pickle.py deleted file mode 100644 index 410fa21087e..00000000000 --- a/python/ql/test/query-tests/Security/lib/pickle.py +++ /dev/null @@ -1,2 +0,0 @@ -def loads(*args, **kwargs): - return None diff --git a/python/ql/test/query-tests/Security/lib/pyOpenSSL/SSL.py b/python/ql/test/query-tests/Security/lib/pyOpenSSL/SSL.py deleted file mode 100644 index eb135d2d5e0..00000000000 --- a/python/ql/test/query-tests/Security/lib/pyOpenSSL/SSL.py +++ /dev/null @@ -1,10 +0,0 @@ -SSLv2_METHOD = 1 -SSLv3_METHOD = 2 -SSLv23_METHOD = 3 -TLSv1_METHOD = 4 -TLSv1_1_METHOD = 5 -TLSv1_2_METHOD = 6 - -class Context(object): - def __init__(self, *args, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/pyOpenSSL/__init__.py b/python/ql/test/query-tests/Security/lib/pyOpenSSL/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/pyramid/__init__.py b/python/ql/test/query-tests/Security/lib/pyramid/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/pyramid/response.py b/python/ql/test/query-tests/Security/lib/pyramid/response.py deleted file mode 100644 index a482ac919ff..00000000000 --- a/python/ql/test/query-tests/Security/lib/pyramid/response.py +++ /dev/null @@ -1,2 +0,0 @@ -class Response(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/pyramid/view.py b/python/ql/test/query-tests/Security/lib/pyramid/view.py deleted file mode 100644 index c7487bf827b..00000000000 --- a/python/ql/test/query-tests/Security/lib/pyramid/view.py +++ /dev/null @@ -1,7 +0,0 @@ -# https://docs.pylonsproject.org/projects/pyramid/en/1.10-branch/_modules/pyramid/view.html#view_config -class view_config(object): - def __init__(self, **settings): - pass - - def __call__(self, wrapped): - pass diff --git a/python/ql/test/query-tests/Security/lib/requests.py b/python/ql/test/query-tests/Security/lib/requests.py deleted file mode 100644 index 01afdbb1208..00000000000 --- a/python/ql/test/query-tests/Security/lib/requests.py +++ /dev/null @@ -1,21 +0,0 @@ - -def get(url, params=None, **kwargs): - pass - -def options(url, **kwargs): - pass - -def head(url, **kwargs): - pass - -def post(url, data=None, json=None, **kwargs): - pass - -def put(url, data=None, **kwargs): - pass - -def patch(url, data=None, **kwargs): - pass - -def delete(url, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/subprocess.py b/python/ql/test/query-tests/Security/lib/subprocess.py deleted file mode 100644 index efb2ba183f0..00000000000 --- a/python/ql/test/query-tests/Security/lib/subprocess.py +++ /dev/null @@ -1,2 +0,0 @@ -def Popen(*args, **kwargs): - return None \ No newline at end of file diff --git a/python/ql/test/query-tests/Security/lib/tg.py b/python/ql/test/query-tests/Security/lib/tg.py deleted file mode 100644 index 976b7427b33..00000000000 --- a/python/ql/test/query-tests/Security/lib/tg.py +++ /dev/null @@ -1,30 +0,0 @@ - -def validate(validators): - def validator(func): - return func - -def expose(*args): - if cond: - return args[0] - def with_template(func): - func - return with_template - -class TGController(object): - pass - -class TurboGearsContextMember(object): - """Member of the TurboGears request context. - Provides access to turbogears context members - like request, response, template context and so on - """ - - def __init__(self, name): - self.__dict__['name'] = name - - def _current_obj(self): - return getattr(context, self.name) - - -request = TurboGearsContextMember(name="request") -response = TurboGearsContextMember(name="response") diff --git a/python/ql/test/query-tests/Security/lib/tornado/__init__.py b/python/ql/test/query-tests/Security/lib/tornado/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/tornado/web.py b/python/ql/test/query-tests/Security/lib/tornado/web.py deleted file mode 100644 index 41b91848992..00000000000 --- a/python/ql/test/query-tests/Security/lib/tornado/web.py +++ /dev/null @@ -1,2 +0,0 @@ -class RequestHandler(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/traceback.py b/python/ql/test/query-tests/Security/lib/traceback.py deleted file mode 100644 index 2a7c5e58847..00000000000 --- a/python/ql/test/query-tests/Security/lib/traceback.py +++ /dev/null @@ -1,2 +0,0 @@ -def format_exc(): - return None \ No newline at end of file diff --git a/python/ql/test/query-tests/Security/lib/twisted/__init__.py b/python/ql/test/query-tests/Security/lib/twisted/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/twisted/web/__init__.py b/python/ql/test/query-tests/Security/lib/twisted/web/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/twisted/web/http.py b/python/ql/test/query-tests/Security/lib/twisted/web/http.py deleted file mode 100644 index c7ac65eaf05..00000000000 --- a/python/ql/test/query-tests/Security/lib/twisted/web/http.py +++ /dev/null @@ -1,2 +0,0 @@ -class Request(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/twisted/web/resource.py b/python/ql/test/query-tests/Security/lib/twisted/web/resource.py deleted file mode 100644 index 1fe186864cb..00000000000 --- a/python/ql/test/query-tests/Security/lib/twisted/web/resource.py +++ /dev/null @@ -1,2 +0,0 @@ -class Resource(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/yaml.py b/python/ql/test/query-tests/Security/lib/yaml.py deleted file mode 100644 index 51fe2fd1d42..00000000000 --- a/python/ql/test/query-tests/Security/lib/yaml.py +++ /dev/null @@ -1,2 +0,0 @@ -def load(s): - pass diff --git a/python/ql/test/query-tests/Security/lib/zope/__init__.py b/python/ql/test/query-tests/Security/lib/zope/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/zope/interface/__init__.py b/python/ql/test/query-tests/Security/lib/zope/interface/__init__.py deleted file mode 100644 index 423c7b58341..00000000000 --- a/python/ql/test/query-tests/Security/lib/zope/interface/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -class Interface(): - pass - - -class implementer: - - def __call__(self, ob): - ... - return ob From ff498f616a9896fb18f693d2cb4c36cd9be4f8d7 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Wed, 10 Apr 2024 13:20:57 +0200 Subject: [PATCH 89/97] C#: Some renaming. --- .../DependencyManager.Nuget.cs | 14 ++--- .../DependencyManager.cs | 61 +++++++++---------- 2 files changed, 37 insertions(+), 38 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs index 40f240f3145..cf9c7253b17 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs @@ -12,14 +12,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { public sealed partial class DependencyManager { - private void RestoreNugetPackages(List allNonBinaryFiles, IEnumerable allProjects, IEnumerable allSolutions, HashSet dllPaths) + private void RestoreNugetPackages(List allNonBinaryFiles, IEnumerable allProjects, IEnumerable allSolutions, HashSet dllLocations) { try { var checkNugetFeedResponsiveness = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.CheckNugetFeedResponsiveness); if (checkNugetFeedResponsiveness && !CheckFeeds(allNonBinaryFiles)) { - DownloadMissingPackages(allNonBinaryFiles, dllPaths, withNugetConfig: false); + DownloadMissingPackages(allNonBinaryFiles, dllLocations, withNugetConfig: false); return; } @@ -55,7 +55,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } nugetPackageDllPaths.ExceptWith(excludedPaths); - dllPaths.UnionWith(nugetPackageDllPaths.Select(p => new AssemblyLookupLocation(p))); + dllLocations.UnionWith(nugetPackageDllPaths.Select(p => new AssemblyLookupLocation(p))); } catch (Exception exc) { @@ -72,10 +72,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching .Paths .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d)) .ToList(); - dllPaths.UnionWith(paths.Select(p => new AssemblyLookupLocation(p))); + dllLocations.UnionWith(paths.Select(p => new AssemblyLookupLocation(p))); LogAllUnusedPackages(dependencies); - DownloadMissingPackages(allNonBinaryFiles, dllPaths); + DownloadMissingPackages(allNonBinaryFiles, dllLocations); } /// @@ -148,7 +148,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString())); } - private void DownloadMissingPackages(List allFiles, ISet dllPaths, bool withNugetConfig = true) + private void DownloadMissingPackages(List allFiles, ISet dllLocations, bool withNugetConfig = true) { var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo); var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames(); @@ -206,7 +206,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString())); - dllPaths.Add(missingPackageDirectory.DirInfo.FullName); + dllLocations.Add(missingPackageDirectory.DirInfo.FullName); } private string[] GetAllNugetConfigs(List allFiles) => allFiles.SelectFileNamesByName("nuget.config").ToArray(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index 8115b0e8ae5..7f8306c2d9d 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -5,7 +5,6 @@ using System.IO; using System.Linq; using System.Security.Cryptography; using System.Text; -using System.Text.RegularExpressions; using System.Threading.Tasks; using Semmle.Util; @@ -91,9 +90,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching this.generatedSources = new(); var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj").ToList(); var allSolutions = allNonBinaryFiles.SelectFileNamesByExtension(".sln").ToList(); - var dllPaths = allFiles.SelectFileNamesByExtension(".dll").Select(x => new AssemblyLookupLocation(x)).ToHashSet(); + var dllLocations = allFiles.SelectFileNamesByExtension(".dll").Select(x => new AssemblyLookupLocation(x)).ToHashSet(); - logger.LogInfo($"Found {allFiles.Count} files, {nonGeneratedSources.Count} source files, {allProjects.Count} project files, {allSolutions.Count} solution files, {dllPaths.Count} DLLs."); + logger.LogInfo($"Found {allFiles.Count} files, {nonGeneratedSources.Count} source files, {allProjects.Count} project files, {allSolutions.Count} solution files, {dllLocations.Count} DLLs."); void startCallback(string s, bool silent) { @@ -122,12 +121,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching throw; } - RestoreNugetPackages(allNonBinaryFiles, allProjects, allSolutions, dllPaths); + RestoreNugetPackages(allNonBinaryFiles, allProjects, allSolutions, dllLocations); // Find DLLs in the .Net / Asp.Net Framework // This needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies. - var frameworkLocations = AddFrameworkDlls(dllPaths); + var frameworkLocations = AddFrameworkDlls(dllLocations); - assemblyCache = new AssemblyCache(dllPaths, frameworkLocations, logger); + assemblyCache = new AssemblyCache(dllLocations, frameworkLocations, logger); AnalyseSolutions(allSolutions); foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename)) @@ -192,7 +191,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching ]); } - private HashSet AddFrameworkDlls(HashSet dllPaths) + private HashSet AddFrameworkDlls(HashSet dllLocations) { var frameworkLocations = new HashSet(); @@ -200,9 +199,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var useSubfolders = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders); if (!string.IsNullOrWhiteSpace(frameworkReferences)) { - RemoveFrameworkNugetPackages(dllPaths); - RemoveNugetPackageReference(FrameworkPackageNames.AspNetCoreFramework, dllPaths); - RemoveNugetPackageReference(FrameworkPackageNames.WindowsDesktopFramework, dllPaths); + RemoveFrameworkNugetPackages(dllLocations); + RemoveNugetPackageReference(FrameworkPackageNames.AspNetCoreFramework, dllLocations); + RemoveNugetPackageReference(FrameworkPackageNames.WindowsDesktopFramework, dllLocations); var frameworkPaths = frameworkReferences.Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries); @@ -214,16 +213,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching continue; } - dllPaths.Add(new AssemblyLookupLocation(path, _ => true, useSubfolders)); + dllLocations.Add(new AssemblyLookupLocation(path, _ => true, useSubfolders)); frameworkLocations.Add(path); } return frameworkLocations; } - AddNetFrameworkDlls(dllPaths, frameworkLocations); - AddAspNetCoreFrameworkDlls(dllPaths, frameworkLocations); - AddMicrosoftWindowsDesktopDlls(dllPaths, frameworkLocations); + AddNetFrameworkDlls(dllLocations, frameworkLocations); + AddAspNetCoreFrameworkDlls(dllLocations, frameworkLocations); + AddMicrosoftWindowsDesktopDlls(dllLocations, frameworkLocations); return frameworkLocations; } @@ -263,7 +262,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet dllPaths, ISet frameworkLocations) + private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet dllLocations, ISet frameworkLocations) { var versionFolders = GetPackageVersionSubDirectories(frameworkPath); if (versionFolders.Length > 1) @@ -279,7 +278,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching selectedFrameworkFolder = frameworkPath; } - dllPaths.Add(selectedFrameworkFolder); + dllLocations.Add(selectedFrameworkFolder); frameworkLocations.Add(selectedFrameworkFolder); logger.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {selectedFrameworkFolder}."); } @@ -292,16 +291,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching .ToArray(); } - private void RemoveFrameworkNugetPackages(ISet dllPaths, int fromIndex = 0) + private void RemoveFrameworkNugetPackages(ISet dllLocations, int fromIndex = 0) { var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks; for (var i = fromIndex; i < packagesInPrioOrder.Length; i++) { - RemoveNugetPackageReference(packagesInPrioOrder[i], dllPaths); + RemoveNugetPackageReference(packagesInPrioOrder[i], dllLocations); } } - private void AddNetFrameworkDlls(ISet dllPaths, ISet frameworkLocations) + private void AddNetFrameworkDlls(ISet dllLocations, ISet frameworkLocations) { // Multiple dotnet framework packages could be present. // The order of the packages is important, we're adding the first one that is present in the nuget cache. @@ -321,8 +320,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching dotnetFrameworkVersionVariantCount += GetPackageVersionSubDirectories(fp.Path!).Length; } - SelectNewestFrameworkPath(frameworkPath.Path, ".NET Framework", dllPaths, frameworkLocations); - RemoveFrameworkNugetPackages(dllPaths, frameworkPath.Index + 1); + SelectNewestFrameworkPath(frameworkPath.Path, ".NET Framework", dllLocations, frameworkLocations); + RemoveFrameworkNugetPackages(dllLocations, frameworkPath.Index + 1); return; } @@ -350,18 +349,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (runtimeLocation is null) { runtimeLocation ??= Runtime.ExecutingRuntime; - dllPaths.Add(new AssemblyLookupLocation(runtimeLocation, name => !name.StartsWith("Semmle."))); + dllLocations.Add(new AssemblyLookupLocation(runtimeLocation, name => !name.StartsWith("Semmle."))); } else { - dllPaths.Add(runtimeLocation); + dllLocations.Add(runtimeLocation); } logger.LogInfo($".NET runtime location selected: {runtimeLocation}"); frameworkLocations.Add(runtimeLocation); } - private void RemoveNugetPackageReference(string packagePrefix, ISet dllPaths) + private void RemoveNugetPackageReference(string packagePrefix, ISet dllLocations) { var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant(); if (packageFolder == null) @@ -370,10 +369,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } var packagePathPrefix = Path.Combine(packageFolder, packagePrefix.ToLowerInvariant()); - var toRemove = dllPaths.Where(s => s.Path.StartsWith(packagePathPrefix, StringComparison.InvariantCultureIgnoreCase)); + var toRemove = dllLocations.Where(s => s.Path.StartsWith(packagePathPrefix, StringComparison.InvariantCultureIgnoreCase)); foreach (var path in toRemove) { - dllPaths.Remove(path); + dllLocations.Remove(path); logger.LogInfo($"Removed reference {path}"); } } @@ -383,7 +382,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return fileContent.IsNewProjectStructureUsed && fileContent.UseAspNetCoreDlls; } - private void AddAspNetCoreFrameworkDlls(ISet dllPaths, ISet frameworkLocations) + private void AddAspNetCoreFrameworkDlls(ISet dllLocations, ISet frameworkLocations) { if (!IsAspNetCoreDetected()) { @@ -393,23 +392,23 @@ namespace Semmle.Extraction.CSharp.DependencyFetching // First try to find ASP.NET Core assemblies in the NuGet packages if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework, packageDirectory) is string aspNetCorePackage) { - SelectNewestFrameworkPath(aspNetCorePackage, "ASP.NET Core", dllPaths, frameworkLocations); + SelectNewestFrameworkPath(aspNetCorePackage, "ASP.NET Core", dllLocations, frameworkLocations); return; } if (Runtime.AspNetCoreRuntime is string aspNetCoreRuntime) { logger.LogInfo($"ASP.NET runtime location selected: {aspNetCoreRuntime}"); - dllPaths.Add(aspNetCoreRuntime); + dllLocations.Add(aspNetCoreRuntime); frameworkLocations.Add(aspNetCoreRuntime); } } - private void AddMicrosoftWindowsDesktopDlls(ISet dllPaths, ISet frameworkLocations) + private void AddMicrosoftWindowsDesktopDlls(ISet dllLocations, ISet frameworkLocations) { if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework, packageDirectory) is string windowsDesktopApp) { - SelectNewestFrameworkPath(windowsDesktopApp, "Windows Desktop App", dllPaths, frameworkLocations); + SelectNewestFrameworkPath(windowsDesktopApp, "Windows Desktop App", dllLocations, frameworkLocations); } } From 4ae25c2d34d7b3c3c72a47a3ea14d2b7d7f7a912 Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Wed, 10 Apr 2024 14:26:00 +0200 Subject: [PATCH 90/97] don't mention arrays in the qhelp for rb/shell-command-constructed-from-input, because there are no array --- .../security/cwe-078/UnsafeShellCommandConstruction.qhelp | 7 ++++++- .../examples/unsafe-shell-command-construction_fixed.rb | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ruby/ql/src/queries/security/cwe-078/UnsafeShellCommandConstruction.qhelp b/ruby/ql/src/queries/security/cwe-078/UnsafeShellCommandConstruction.qhelp index 4231f7cb0b4..8cedb57c277 100644 --- a/ruby/ql/src/queries/security/cwe-078/UnsafeShellCommandConstruction.qhelp +++ b/ruby/ql/src/queries/security/cwe-078/UnsafeShellCommandConstruction.qhelp @@ -20,10 +20,15 @@

- If possible, provide the dynamic arguments to the shell as an array + If possible, avoid concatenating shell strings to APIs such as system(..) to avoid interpretation by the shell.

+

+ Instead, provide the arguments to the shell command as separate arguments to the + API, such as system("echo", arg1, arg2). +

+

Alternatively, if the shell command must be constructed dynamically, then add code to ensure that special characters diff --git a/ruby/ql/src/queries/security/cwe-078/examples/unsafe-shell-command-construction_fixed.rb b/ruby/ql/src/queries/security/cwe-078/examples/unsafe-shell-command-construction_fixed.rb index cb8730eee09..b055f94eba9 100644 --- a/ruby/ql/src/queries/security/cwe-078/examples/unsafe-shell-command-construction_fixed.rb +++ b/ruby/ql/src/queries/security/cwe-078/examples/unsafe-shell-command-construction_fixed.rb @@ -1,6 +1,6 @@ module Utils def download(path) - # using an array to call `system` is safe + # using an API that doesn't interpret the path as a shell command system("wget", path) # OK end end \ No newline at end of file From ef68e3344980006eec8eabef044053d3b69d7452 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Wed, 10 Apr 2024 14:48:02 +0200 Subject: [PATCH 91/97] C#: Address review comments. --- .../AssemblyLookupLocation.cs | 37 ++++++++++--------- .../global.json | 5 --- 2 files changed, 19 insertions(+), 23 deletions(-) delete mode 100644 csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/global.json diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs index 533b547e45d..98f43b9dcb7 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs @@ -10,11 +10,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// Used to represent a path to an assembly or a directory containing assemblies /// and a selector function to determine which files to include, when indexing the assemblies. ///

- internal sealed class AssemblyLookupLocation(string p, Func includeFileName, bool indexSubdirectories = true) + internal sealed class AssemblyLookupLocation(string path, Func includeFileName, bool indexSubdirectories = true) { - public string Path => p; + public string Path => path; - public AssemblyLookupLocation(string p) : this(p, _ => true) { } + public AssemblyLookupLocation(string path) : this(path, _ => true) { } public static implicit operator AssemblyLookupLocation(string path) => new(path); @@ -23,15 +23,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// and adds them to the a list of assembly names to index. /// Indexing is performed at a later stage. This only collects the names. /// - /// The directory to index. + /// List of dlls to index. + /// Logger. private void AddReferenceDirectory(List dllsToIndex, ILogger logger) { try { - var dlls = new DirectoryInfo(p).EnumerateFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = indexSubdirectories, MatchCasing = MatchCasing.CaseInsensitive, AttributesToSkip = FileAttributes.None }); + var dlls = new DirectoryInfo(path).EnumerateFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = indexSubdirectories, MatchCasing = MatchCasing.CaseInsensitive, AttributesToSkip = FileAttributes.None }); if (!dlls.Any()) { - logger.LogWarning($"AssemblyLookupLocation: No DLLs found in the path '{p}'."); + logger.LogWarning($"AssemblyLookupLocation: No DLLs found in the path '{path}'."); return; } foreach (var dll in dlls) @@ -48,47 +49,47 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } catch (Exception e) { - logger.LogError($"AssemblyLookupLocation: Error while searching for DLLs in '{p}': {e.Message}"); + logger.LogError($"AssemblyLookupLocation: Error while searching for DLLs in '{path}': {e.Message}"); } } /// - /// Returns a list of paths to all assemblies in `p` that should be indexed. + /// Returns a list of paths to all assemblies in `path` that should be indexed. /// /// Logger public List GetDlls(ILogger logger) { var dllsToIndex = new List(); - if (File.Exists(p)) + if (File.Exists(path)) { - if (includeFileName(System.IO.Path.GetFileName(p))) + if (includeFileName(System.IO.Path.GetFileName(path))) { - dllsToIndex.Add(p); + dllsToIndex.Add(path); } else { - logger.LogInfo($"AssemblyLookupLocation: Skipping {p}."); + logger.LogInfo($"AssemblyLookupLocation: Skipping {path}."); } return dllsToIndex; } - if (Directory.Exists(p)) + if (Directory.Exists(path)) { - logger.LogInfo($"AssemblyLookupLocation: Finding reference DLLs in {p}..."); + logger.LogInfo($"AssemblyLookupLocation: Finding reference DLLs in {path}..."); AddReferenceDirectory(dllsToIndex, logger); } else { - logger.LogInfo("AssemblyLookupLocation: Path not found: " + p); + logger.LogInfo("AssemblyLookupLocation: Path not found: " + path); } return dllsToIndex; } public override bool Equals(object? obj) => - obj is AssemblyLookupLocation ap && p.Equals(ap.Path); + obj is AssemblyLookupLocation ap && path.Equals(ap.Path); - public override int GetHashCode() => p.GetHashCode(); + public override int GetHashCode() => path.GetHashCode(); - public override string ToString() => p; + public override string ToString() => path; } } diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/global.json b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/global.json deleted file mode 100644 index d54915e8d4d..00000000000 --- a/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/global.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "sdk": { - "version": "8.0.101" - } -} From d98ed2d3cf23c820a0827d24ea27bfa9c2313fa7 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Wed, 10 Apr 2024 14:17:04 +0100 Subject: [PATCH 92/97] C++: Add alias and side effect models for more iterator functions. --- .../cpp/models/implementations/Iterator.qll | 151 +++++++++++++++++- 1 file changed, 146 insertions(+), 5 deletions(-) diff --git a/cpp/ql/lib/semmle/code/cpp/models/implementations/Iterator.qll b/cpp/ql/lib/semmle/code/cpp/models/implementations/Iterator.qll index 47e4d5f4f0c..f8d31b5583f 100644 --- a/cpp/ql/lib/semmle/code/cpp/models/implementations/Iterator.qll +++ b/cpp/ql/lib/semmle/code/cpp/models/implementations/Iterator.qll @@ -123,7 +123,7 @@ class IteratorCrementNonMemberOperator extends Operator { } private class IteratorCrementNonMemberOperatorModel extends IteratorCrementNonMemberOperator, - DataFlowFunction + DataFlowFunction, SideEffectFunction, AliasFunction { override predicate hasDataFlow(FunctionInput input, FunctionOutput output) { input = getIteratorArgumentInput(this, 0) and @@ -131,6 +131,26 @@ private class IteratorCrementNonMemberOperatorModel extends IteratorCrementNonMe or input.isParameterDeref(0) and output.isReturnValueDeref() } + + override predicate hasOnlySpecificReadSideEffects() { any() } + + override predicate hasOnlySpecificWriteSideEffects() { any() } + + override predicate hasSpecificReadSideEffect(ParameterIndex i, boolean buffer) { + i = 0 and buffer = false + } + + override predicate hasSpecificWriteSideEffect(ParameterIndex i, boolean buffer, boolean mustWrite) { + // `buffer` must be `true` or `mustWrite` must be `false` to ensure that + // the IR alias analysis doesn't think that `it++` doesn't completely override + // the value of the iterator. Ideally, the IR alias analysis shouldn't deal with + // iterators, but this is necessary for taintflow to get any results in our iterator tests. + i = 0 and buffer = false and mustWrite = false + } + + override predicate parameterNeverEscapes(int index) { none() } + + override predicate parameterEscapesOnlyViaReturn(int index) { index = 0 } } /** @@ -146,7 +166,7 @@ class IteratorCrementMemberOperator extends MemberFunction { } private class IteratorCrementMemberOperatorModel extends IteratorCrementMemberOperator, - DataFlowFunction, TaintFunction + DataFlowFunction, TaintFunction, SideEffectFunction, AliasFunction { override predicate hasDataFlow(FunctionInput input, FunctionOutput output) { input.isQualifierAddress() and @@ -163,6 +183,26 @@ private class IteratorCrementMemberOperatorModel extends IteratorCrementMemberOp input.isQualifierObject() and output.isReturnValueDeref() } + + override predicate hasOnlySpecificReadSideEffects() { any() } + + override predicate hasOnlySpecificWriteSideEffects() { any() } + + override predicate hasSpecificReadSideEffect(ParameterIndex i, boolean buffer) { + i = -1 and buffer = false + } + + override predicate hasSpecificWriteSideEffect(ParameterIndex i, boolean buffer, boolean mustWrite) { + // `buffer` must be `true` or `mustWrite` must be `false` to ensure that + // the IR alias analysis doesn't think that `it++` doesn't completely override + // the value of the iterator. Ideally, the IR alias analysis shouldn't deal with + // iterators, but this is necessary for taintflow to get any results in our iterator tests. + i = -1 and buffer = false and mustWrite = false + } + + override predicate parameterNeverEscapes(int index) { index = -1 } + + override predicate parameterEscapesOnlyViaReturn(int index) { none() } } /** @@ -332,7 +372,7 @@ class IteratorAssignArithmeticOperator extends Function { * non-member and member versions, use `IteratorPointerDereferenceOperator`. */ class IteratorPointerDereferenceMemberOperator extends MemberFunction, TaintFunction, - IteratorReferenceFunction + IteratorReferenceFunction, AliasFunction, SideEffectFunction { IteratorPointerDereferenceMemberOperator() { this.getClassAndName("operator*") instanceof Iterator @@ -345,6 +385,18 @@ class IteratorPointerDereferenceMemberOperator extends MemberFunction, TaintFunc input.isReturnValueDeref() and output.isQualifierObject() } + + override predicate parameterNeverEscapes(int index) { index = -1 } + + override predicate parameterEscapesOnlyViaReturn(int index) { none() } + + override predicate hasOnlySpecificReadSideEffects() { any() } + + override predicate hasOnlySpecificWriteSideEffects() { any() } + + override predicate hasSpecificReadSideEffect(ParameterIndex i, boolean buffer) { + i = -1 and buffer = false + } } /** @@ -361,7 +413,7 @@ class IteratorPointerDereferenceNonMemberOperator extends Operator, IteratorRefe } private class IteratorPointerDereferenceNonMemberOperatorModel extends IteratorPointerDereferenceNonMemberOperator, - TaintFunction + TaintFunction, AliasFunction, SideEffectFunction { override predicate hasTaintFlow(FunctionInput input, FunctionOutput output) { input = getIteratorArgumentInput(this, 0) and @@ -370,6 +422,18 @@ private class IteratorPointerDereferenceNonMemberOperatorModel extends IteratorP input.isReturnValueDeref() and output.isParameterDeref(0) } + + override predicate parameterNeverEscapes(int index) { index = 0 } + + override predicate parameterEscapesOnlyViaReturn(int index) { none() } + + override predicate hasOnlySpecificReadSideEffects() { any() } + + override predicate hasOnlySpecificWriteSideEffects() { any() } + + override predicate hasSpecificReadSideEffect(ParameterIndex i, boolean buffer) { + i = 0 and buffer = false + } } /** @@ -420,6 +484,71 @@ class IteratorAssignmentMemberOperator extends MemberFunction { } } +/** + * A member `operator==` or `operator!=` function for an iterator type. + * + * Note that this class _only_ matches member functions. To find both + * non-member and member versions, use `IteratorLogicalOperator`. + */ +class IteratorLogicalMemberOperator extends MemberFunction { + IteratorLogicalMemberOperator() { + this.getClassAndName(["operator!=", "operator=="]) instanceof Iterator + } +} + +private class IteratorLogicalMemberOperatorModel extends IteratorLogicalMemberOperator, + AliasFunction, SideEffectFunction +{ + override predicate parameterNeverEscapes(int index) { index = [-1, 0] } + + override predicate parameterEscapesOnlyViaReturn(int index) { none() } + + override predicate hasOnlySpecificReadSideEffects() { any() } + + override predicate hasOnlySpecificWriteSideEffects() { any() } + + override predicate hasSpecificReadSideEffect(ParameterIndex i, boolean buffer) { + i = -1 and buffer = false + } +} + +/** + * A member `operator==` or `operator!=` function for an iterator type. + * + * Note that this class _only_ matches non-member functions. To find both + * non-member and member versions, use `IteratorLogicalOperator`. + */ +class IteratorLogicalNonMemberOperator extends Function { + IteratorLogicalNonMemberOperator() { + this.hasName(["operator!=", "operator=="]) and + exists(getIteratorArgumentInput(this, 0)) and + exists(getIteratorArgumentInput(this, 1)) + } +} + +private class IteratorLogicalNonMemberOperatorModel extends IteratorLogicalNonMemberOperator, + AliasFunction, SideEffectFunction +{ + override predicate parameterNeverEscapes(int index) { index = [0, 1] } + + override predicate parameterEscapesOnlyViaReturn(int index) { none() } + + override predicate hasOnlySpecificReadSideEffects() { any() } + + override predicate hasOnlySpecificWriteSideEffects() { any() } +} + +/** + * A (member or non-member) `operator==` or `operator!=` function for an iterator type. + */ +class IteratorLogicalOperator extends Function { + IteratorLogicalOperator() { + this instanceof IteratorLogicalNonMemberOperator + or + this instanceof IteratorLogicalMemberOperator + } +} + /** * An `operator=` member function of an iterator class that is not a copy or move assignment * operator. @@ -428,12 +557,24 @@ class IteratorAssignmentMemberOperator extends MemberFunction { * `operator*` and use their own `operator=` to assign to the container. */ private class IteratorAssignmentMemberOperatorModel extends IteratorAssignmentMemberOperator, - TaintFunction + TaintFunction, SideEffectFunction, AliasFunction { override predicate hasTaintFlow(FunctionInput input, FunctionOutput output) { input.isParameterDeref(0) and output.isQualifierObject() } + + override predicate hasOnlySpecificReadSideEffects() { any() } + + override predicate hasOnlySpecificWriteSideEffects() { any() } + + override predicate hasSpecificWriteSideEffect(ParameterIndex i, boolean buffer, boolean mustWrite) { + i = -1 and buffer = false and mustWrite = false + } + + override predicate parameterNeverEscapes(int index) { index = 0 } + + override predicate parameterEscapesOnlyViaReturn(int index) { index = -1 } } /** From a7c98e3d94aed08bfbe27fbc22d691af2b38b531 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Wed, 10 Apr 2024 14:17:12 +0100 Subject: [PATCH 93/97] C++: Accept test changes. --- .../library-tests/ir/ir/aliased_ir.expected | 533 ++++++++---------- .../ir/ir/operand_locations.expected | 488 +++++++--------- .../test/library-tests/ir/ir/raw_ir.expected | 400 ++++++------- 3 files changed, 637 insertions(+), 784 deletions(-) diff --git a/cpp/ql/test/library-tests/ir/ir/aliased_ir.expected b/cpp/ql/test/library-tests/ir/ir/aliased_ir.expected index 39eccf0139f..4c183b525c5 100644 --- a/cpp/ql/test/library-tests/ir/ir/aliased_ir.expected +++ b/cpp/ql/test/library-tests/ir/ir/aliased_ir.expected @@ -7351,8 +7351,8 @@ ir.cpp: #-----| Goto -> Block 1 # 1127| Block 1 -# 1127| m1127_19(iterator) = Phi : from 0:m1127_12, from 4:m1127_50 -# 1127| m1127_20(unknown) = Phi : from 0:~m1126_4, from 4:~m1127_47 +# 1127| m1127_19(iterator) = Phi : from 0:m1127_12, from 4:m1127_44 +# 1127| m1127_20(unknown) = Phi : from 0:~m1126_4, from 4:~m1127_27 # 1127| r1127_21(glval>) = VariableAddress[(__begin)] : #-----| r0_5(glval>) = Convert : r1127_21 # 1127| r1127_22(glval) = FunctionAddress[operator!=] : @@ -7370,26 +7370,22 @@ ir.cpp: # 1127| m1127_29(iterator) = Chi : total:m0_7, partial:m1127_28 #-----| r0_11(iterator) = Load[#temp0:0] : &:r0_6, m1127_29 # 1127| r1127_30(bool) = Call[operator!=] : func:r1127_22, this:r0_5, 0:r0_11 -# 1127| m1127_31(unknown) = ^CallSideEffect : ~m1127_27 -# 1127| m1127_32(unknown) = Chi : total:m1127_27, partial:m1127_31 #-----| v0_12(void) = ^IndirectReadSideEffect[-1] : &:r0_5, m1127_19 -# 1127| v1127_33(void) = ConditionalBranch : r1127_30 +# 1127| v1127_31(void) = ConditionalBranch : r1127_30 #-----| False -> Block 5 #-----| True -> Block 2 # 1127| Block 2 -# 1127| r1127_34(glval) = VariableAddress[e] : -# 1127| r1127_35(glval>) = VariableAddress[(__begin)] : -#-----| r0_13(glval>) = Convert : r1127_35 -# 1127| r1127_36(glval) = FunctionAddress[operator*] : -# 1127| r1127_37(int &) = Call[operator*] : func:r1127_36, this:r0_13 -# 1127| m1127_38(unknown) = ^CallSideEffect : ~m1127_32 -# 1127| m1127_39(unknown) = Chi : total:m1127_32, partial:m1127_38 +# 1127| r1127_32(glval) = VariableAddress[e] : +# 1127| r1127_33(glval>) = VariableAddress[(__begin)] : +#-----| r0_13(glval>) = Convert : r1127_33 +# 1127| r1127_34(glval) = FunctionAddress[operator*] : +# 1127| r1127_35(int &) = Call[operator*] : func:r1127_34, this:r0_13 #-----| v0_14(void) = ^IndirectReadSideEffect[-1] : &:r0_13, m1127_19 -# 1127| r1127_40(int) = Load[?] : &:r1127_37, ~m1127_39 -# 1127| m1127_41(int) = Store[e] : &:r1127_34, r1127_40 +# 1127| r1127_36(int) = Load[?] : &:r1127_35, ~m1127_27 +# 1127| m1127_37(int) = Store[e] : &:r1127_32, r1127_36 # 1128| r1128_1(glval) = VariableAddress[e] : -# 1128| r1128_2(int) = Load[e] : &:r1128_1, m1127_41 +# 1128| r1128_2(int) = Load[e] : &:r1128_1, m1127_37 # 1128| r1128_3(int) = Constant[0] : # 1128| r1128_4(bool) = CompareGT : r1128_2, r1128_3 # 1128| v1128_5(void) = ConditionalBranch : r1128_4 @@ -7401,16 +7397,14 @@ ir.cpp: #-----| Goto -> Block 4 # 1127| Block 4 -# 1127| v1127_42(void) = NoOp : -# 1127| r1127_43(glval>) = VariableAddress[(__begin)] : -# 1127| r1127_44(glval) = FunctionAddress[operator++] : -# 1127| r1127_45(iterator &) = Call[operator++] : func:r1127_44, this:r1127_43 -# 1127| m1127_46(unknown) = ^CallSideEffect : ~m1127_39 -# 1127| m1127_47(unknown) = Chi : total:m1127_39, partial:m1127_46 -# 1127| v1127_48(void) = ^IndirectReadSideEffect[-1] : &:r1127_43, m1127_19 -# 1127| m1127_49(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r1127_43 -# 1127| m1127_50(iterator) = Chi : total:m1127_19, partial:m1127_49 -# 1127| r1127_51(glval>) = CopyValue : r1127_45 +# 1127| v1127_38(void) = NoOp : +# 1127| r1127_39(glval>) = VariableAddress[(__begin)] : +# 1127| r1127_40(glval) = FunctionAddress[operator++] : +# 1127| r1127_41(iterator &) = Call[operator++] : func:r1127_40, this:r1127_39 +# 1127| v1127_42(void) = ^IndirectReadSideEffect[-1] : &:r1127_39, m1127_19 +# 1127| m1127_43(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r1127_39 +# 1127| m1127_44(iterator) = Chi : total:m1127_19, partial:m1127_43 +# 1127| r1127_45(glval>) = CopyValue : r1127_41 #-----| Goto (back edge) -> Block 1 # 1133| Block 5 @@ -7439,8 +7433,8 @@ ir.cpp: #-----| Goto -> Block 6 # 1133| Block 6 -# 1133| m1133_19(iterator) = Phi : from 5:m1133_12, from 7:m1133_41 -# 1133| m1133_20(unknown) = Phi : from 5:~m1127_32, from 7:~m1133_38 +# 1133| m1133_19(iterator) = Phi : from 5:m1133_12, from 7:m1133_37 +# 1133| m1133_20(unknown) = Phi : from 5:~m1127_27, from 7:~m1133_27 # 1133| r1133_21(glval>) = VariableAddress[(__begin)] : #-----| r0_19(glval>) = Convert : r1133_21 # 1133| r1133_22(glval) = FunctionAddress[operator!=] : @@ -7458,41 +7452,35 @@ ir.cpp: # 1133| m1133_29(iterator) = Chi : total:m0_21, partial:m1133_28 #-----| r0_25(iterator) = Load[#temp0:0] : &:r0_20, m1133_29 # 1133| r1133_30(bool) = Call[operator!=] : func:r1133_22, this:r0_19, 0:r0_25 -# 1133| m1133_31(unknown) = ^CallSideEffect : ~m1133_27 -# 1133| m1133_32(unknown) = Chi : total:m1133_27, partial:m1133_31 #-----| v0_26(void) = ^IndirectReadSideEffect[-1] : &:r0_19, m1133_19 -# 1133| v1133_33(void) = ConditionalBranch : r1133_30 +# 1133| v1133_31(void) = ConditionalBranch : r1133_30 #-----| False -> Block 10 #-----| True -> Block 8 # 1133| Block 7 -# 1133| r1133_34(glval>) = VariableAddress[(__begin)] : -# 1133| r1133_35(glval) = FunctionAddress[operator++] : -# 1133| r1133_36(iterator &) = Call[operator++] : func:r1133_35, this:r1133_34 -# 1133| m1133_37(unknown) = ^CallSideEffect : ~m1133_48 -# 1133| m1133_38(unknown) = Chi : total:m1133_48, partial:m1133_37 -# 1133| v1133_39(void) = ^IndirectReadSideEffect[-1] : &:r1133_34, m1133_19 -# 1133| m1133_40(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r1133_34 -# 1133| m1133_41(iterator) = Chi : total:m1133_19, partial:m1133_40 -# 1133| r1133_42(glval>) = CopyValue : r1133_36 +# 1133| r1133_32(glval>) = VariableAddress[(__begin)] : +# 1133| r1133_33(glval) = FunctionAddress[operator++] : +# 1133| r1133_34(iterator &) = Call[operator++] : func:r1133_33, this:r1133_32 +# 1133| v1133_35(void) = ^IndirectReadSideEffect[-1] : &:r1133_32, m1133_19 +# 1133| m1133_36(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r1133_32 +# 1133| m1133_37(iterator) = Chi : total:m1133_19, partial:m1133_36 +# 1133| r1133_38(glval>) = CopyValue : r1133_34 #-----| Goto (back edge) -> Block 6 # 1133| Block 8 -# 1133| r1133_43(glval) = VariableAddress[e] : -# 1133| r1133_44(glval>) = VariableAddress[(__begin)] : -#-----| r0_27(glval>) = Convert : r1133_44 -# 1133| r1133_45(glval) = FunctionAddress[operator*] : -# 1133| r1133_46(int &) = Call[operator*] : func:r1133_45, this:r0_27 -# 1133| m1133_47(unknown) = ^CallSideEffect : ~m1133_32 -# 1133| m1133_48(unknown) = Chi : total:m1133_32, partial:m1133_47 +# 1133| r1133_39(glval) = VariableAddress[e] : +# 1133| r1133_40(glval>) = VariableAddress[(__begin)] : +#-----| r0_27(glval>) = Convert : r1133_40 +# 1133| r1133_41(glval) = FunctionAddress[operator*] : +# 1133| r1133_42(int &) = Call[operator*] : func:r1133_41, this:r0_27 #-----| v0_28(void) = ^IndirectReadSideEffect[-1] : &:r0_27, m1133_19 -# 1133| r1133_49(glval) = CopyValue : r1133_46 -# 1133| r1133_50(glval) = Convert : r1133_49 -# 1133| r1133_51(int &) = CopyValue : r1133_50 -# 1133| m1133_52(int &) = Store[e] : &:r1133_43, r1133_51 +# 1133| r1133_43(glval) = CopyValue : r1133_42 +# 1133| r1133_44(glval) = Convert : r1133_43 +# 1133| r1133_45(int &) = CopyValue : r1133_44 +# 1133| m1133_46(int &) = Store[e] : &:r1133_39, r1133_45 # 1134| r1134_1(glval) = VariableAddress[e] : -# 1134| r1134_2(int &) = Load[e] : &:r1134_1, m1133_52 -# 1134| r1134_3(int) = Load[?] : &:r1134_2, ~m1133_48 +# 1134| r1134_2(int &) = Load[e] : &:r1134_1, m1133_46 +# 1134| r1134_3(int) = Load[?] : &:r1134_2, ~m1133_27 # 1134| r1134_4(int) = Constant[5] : # 1134| r1134_5(bool) = CompareLT : r1134_3, r1134_4 # 1134| v1134_6(void) = ConditionalBranch : r1134_5 @@ -7504,13 +7492,12 @@ ir.cpp: #-----| Goto -> Block 10 # 1137| Block 10 -# 1137| m1137_1(unknown) = Phi : from 6:~m1133_32, from 9:~m1133_48 -# 1137| v1137_2(void) = NoOp : -# 1138| v1138_1(void) = NoOp : -# 1126| v1126_9(void) = ReturnIndirection[v] : &:r1126_7, m1126_8 -# 1126| v1126_10(void) = ReturnVoid : -# 1126| v1126_11(void) = AliasedUse : ~m1137_1 -# 1126| v1126_12(void) = ExitFunction : +# 1137| v1137_1(void) = NoOp : +# 1138| v1138_1(void) = NoOp : +# 1126| v1126_9(void) = ReturnIndirection[v] : &:r1126_7, m1126_8 +# 1126| v1126_10(void) = ReturnVoid : +# 1126| v1126_11(void) = AliasedUse : ~m1133_27 +# 1126| v1126_12(void) = ExitFunction : # 1157| int AsmStmt(int) # 1157| Block 0 @@ -13555,8 +13542,8 @@ ir.cpp: #-----| Goto -> Block 8 # 2215| Block 8 -# 2215| m2215_30(iterator) = Phi : from 7:m2215_23, from 9:m2215_68 -# 2215| m2215_31(unknown) = Phi : from 7:~m2215_11, from 9:~m2215_65 +# 2215| m2215_30(iterator) = Phi : from 7:m2215_23, from 9:m2215_62 +# 2215| m2215_31(unknown) = Phi : from 7:~m2215_11, from 9:~m2215_53 # 2215| r2215_32(glval>) = VariableAddress[(__begin)] : #-----| r0_7(glval>) = Convert : r2215_32 # 2215| r2215_33(glval) = FunctionAddress[operator!=] : @@ -13574,50 +13561,44 @@ ir.cpp: # 2215| m2215_40(iterator) = Chi : total:m0_9, partial:m2215_39 #-----| r0_13(iterator) = Load[#temp0:0] : &:r0_8, m2215_40 # 2215| r2215_41(bool) = Call[operator!=] : func:r2215_33, this:r0_7, 0:r0_13 -# 2215| m2215_42(unknown) = ^CallSideEffect : ~m2215_38 -# 2215| m2215_43(unknown) = Chi : total:m2215_38, partial:m2215_42 #-----| v0_14(void) = ^IndirectReadSideEffect[-1] : &:r0_7, m2215_30 -# 2215| v2215_44(void) = ConditionalBranch : r2215_41 +# 2215| v2215_42(void) = ConditionalBranch : r2215_41 #-----| False -> Block 10 #-----| True -> Block 9 # 2215| Block 9 -# 2215| r2215_45(glval) = VariableAddress[y] : -# 2215| r2215_46(glval>) = VariableAddress[(__begin)] : -#-----| r0_15(glval>) = Convert : r2215_46 -# 2215| r2215_47(glval) = FunctionAddress[operator*] : -# 2215| r2215_48(ClassWithDestructor &) = Call[operator*] : func:r2215_47, this:r0_15 -# 2215| m2215_49(unknown) = ^CallSideEffect : ~m2215_43 -# 2215| m2215_50(unknown) = Chi : total:m2215_43, partial:m2215_49 +# 2215| r2215_43(glval) = VariableAddress[y] : +# 2215| r2215_44(glval>) = VariableAddress[(__begin)] : +#-----| r0_15(glval>) = Convert : r2215_44 +# 2215| r2215_45(glval) = FunctionAddress[operator*] : +# 2215| r2215_46(ClassWithDestructor &) = Call[operator*] : func:r2215_45, this:r0_15 #-----| v0_16(void) = ^IndirectReadSideEffect[-1] : &:r0_15, m2215_30 -# 2215| r2215_51(ClassWithDestructor) = Load[?] : &:r2215_48, ~m2215_50 -# 2215| m2215_52(ClassWithDestructor) = Store[y] : &:r2215_45, r2215_51 +# 2215| r2215_47(ClassWithDestructor) = Load[?] : &:r2215_46, ~m2215_38 +# 2215| m2215_48(ClassWithDestructor) = Store[y] : &:r2215_43, r2215_47 # 2216| r2216_1(glval) = VariableAddress[y] : # 2216| r2216_2(glval) = FunctionAddress[set_x] : # 2216| r2216_3(char) = Constant[97] : # 2216| v2216_4(void) = Call[set_x] : func:r2216_2, this:r2216_1, 0:r2216_3 -# 2216| m2216_5(unknown) = ^CallSideEffect : ~m2215_50 -# 2216| m2216_6(unknown) = Chi : total:m2215_50, partial:m2216_5 -# 2216| v2216_7(void) = ^IndirectReadSideEffect[-1] : &:r2216_1, m2215_52 +# 2216| m2216_5(unknown) = ^CallSideEffect : ~m2215_38 +# 2216| m2216_6(unknown) = Chi : total:m2215_38, partial:m2216_5 +# 2216| v2216_7(void) = ^IndirectReadSideEffect[-1] : &:r2216_1, m2215_48 # 2216| m2216_8(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2216_1 -# 2216| m2216_9(ClassWithDestructor) = Chi : total:m2215_52, partial:m2216_8 -# 2215| r2215_53(glval) = VariableAddress[y] : -# 2215| r2215_54(glval) = FunctionAddress[~ClassWithDestructor] : -# 2215| v2215_55(void) = Call[~ClassWithDestructor] : func:r2215_54, this:r2215_53 -# 2215| m2215_56(unknown) = ^CallSideEffect : ~m2216_6 -# 2215| m2215_57(unknown) = Chi : total:m2216_6, partial:m2215_56 -# 2215| v2215_58(void) = ^IndirectReadSideEffect[-1] : &:r2215_53, m2216_9 -# 2215| m2215_59(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2215_53 -# 2215| m2215_60(ClassWithDestructor) = Chi : total:m2216_9, partial:m2215_59 -# 2215| r2215_61(glval>) = VariableAddress[(__begin)] : -# 2215| r2215_62(glval) = FunctionAddress[operator++] : -# 2215| r2215_63(iterator &) = Call[operator++] : func:r2215_62, this:r2215_61 -# 2215| m2215_64(unknown) = ^CallSideEffect : ~m2215_57 -# 2215| m2215_65(unknown) = Chi : total:m2215_57, partial:m2215_64 -# 2215| v2215_66(void) = ^IndirectReadSideEffect[-1] : &:r2215_61, m2215_30 -# 2215| m2215_67(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2215_61 -# 2215| m2215_68(iterator) = Chi : total:m2215_30, partial:m2215_67 -# 2215| r2215_69(glval>) = CopyValue : r2215_63 +# 2216| m2216_9(ClassWithDestructor) = Chi : total:m2215_48, partial:m2216_8 +# 2215| r2215_49(glval) = VariableAddress[y] : +# 2215| r2215_50(glval) = FunctionAddress[~ClassWithDestructor] : +# 2215| v2215_51(void) = Call[~ClassWithDestructor] : func:r2215_50, this:r2215_49 +# 2215| m2215_52(unknown) = ^CallSideEffect : ~m2216_6 +# 2215| m2215_53(unknown) = Chi : total:m2216_6, partial:m2215_52 +# 2215| v2215_54(void) = ^IndirectReadSideEffect[-1] : &:r2215_49, m2216_9 +# 2215| m2215_55(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2215_49 +# 2215| m2215_56(ClassWithDestructor) = Chi : total:m2216_9, partial:m2215_55 +# 2215| r2215_57(glval>) = VariableAddress[(__begin)] : +# 2215| r2215_58(glval) = FunctionAddress[operator++] : +# 2215| r2215_59(iterator &) = Call[operator++] : func:r2215_58, this:r2215_57 +# 2215| v2215_60(void) = ^IndirectReadSideEffect[-1] : &:r2215_57, m2215_30 +# 2215| m2215_61(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2215_57 +# 2215| m2215_62(iterator) = Chi : total:m2215_30, partial:m2215_61 +# 2215| r2215_63(glval>) = CopyValue : r2215_59 #-----| Goto (back edge) -> Block 8 # 2218| Block 10 @@ -13630,8 +13611,8 @@ ir.cpp: # 2218| m2218_7(ClassWithDestructor) = Store[#temp2218:45] : &:r2218_4, r2218_6 # 2218| r2218_8(ClassWithDestructor) = Load[#temp2218:45] : &:r2218_4, m2218_7 # 2218| v2218_9(void) = Call[vector] : func:r2218_3, this:r2218_1, 0:r2218_8 -# 2218| m2218_10(unknown) = ^CallSideEffect : ~m2215_43 -# 2218| m2218_11(unknown) = Chi : total:m2215_43, partial:m2218_10 +# 2218| m2218_10(unknown) = ^CallSideEffect : ~m2215_38 +# 2218| m2218_11(unknown) = Chi : total:m2215_38, partial:m2218_10 # 2218| m2218_12(vector) = ^IndirectMayWriteSideEffect[-1] : &:r2218_1 # 2218| m2218_13(vector) = Chi : total:m2218_2, partial:m2218_12 # 2218| r2218_14(glval &>) = VariableAddress[(__range)] : @@ -13659,8 +13640,8 @@ ir.cpp: #-----| Goto -> Block 11 # 2218| Block 11 -# 2218| m2218_30(iterator) = Phi : from 10:m2218_23, from 14:m2218_84 -# 2218| m2218_31(unknown) = Phi : from 10:~m2218_11, from 14:~m2218_81 +# 2218| m2218_30(iterator) = Phi : from 10:m2218_23, from 14:m2218_78 +# 2218| m2218_31(unknown) = Phi : from 10:~m2218_11, from 14:~m2218_69 # 2218| r2218_32(glval>) = VariableAddress[(__begin)] : #-----| r0_23(glval>) = Convert : r2218_32 # 2218| r2218_33(glval) = FunctionAddress[operator!=] : @@ -13678,33 +13659,29 @@ ir.cpp: # 2218| m2218_40(iterator) = Chi : total:m0_25, partial:m2218_39 #-----| r0_29(iterator) = Load[#temp0:0] : &:r0_24, m2218_40 # 2218| r2218_41(bool) = Call[operator!=] : func:r2218_33, this:r0_23, 0:r0_29 -# 2218| m2218_42(unknown) = ^CallSideEffect : ~m2218_38 -# 2218| m2218_43(unknown) = Chi : total:m2218_38, partial:m2218_42 #-----| v0_30(void) = ^IndirectReadSideEffect[-1] : &:r0_23, m2218_30 -# 2218| v2218_44(void) = ConditionalBranch : r2218_41 +# 2218| v2218_42(void) = ConditionalBranch : r2218_41 #-----| False -> Block 15 #-----| True -> Block 12 # 2218| Block 12 -# 2218| r2218_45(glval) = VariableAddress[y] : -# 2218| r2218_46(glval>) = VariableAddress[(__begin)] : -#-----| r0_31(glval>) = Convert : r2218_46 -# 2218| r2218_47(glval) = FunctionAddress[operator*] : -# 2218| r2218_48(ClassWithDestructor &) = Call[operator*] : func:r2218_47, this:r0_31 -# 2218| m2218_49(unknown) = ^CallSideEffect : ~m2218_43 -# 2218| m2218_50(unknown) = Chi : total:m2218_43, partial:m2218_49 +# 2218| r2218_43(glval) = VariableAddress[y] : +# 2218| r2218_44(glval>) = VariableAddress[(__begin)] : +#-----| r0_31(glval>) = Convert : r2218_44 +# 2218| r2218_45(glval) = FunctionAddress[operator*] : +# 2218| r2218_46(ClassWithDestructor &) = Call[operator*] : func:r2218_45, this:r0_31 #-----| v0_32(void) = ^IndirectReadSideEffect[-1] : &:r0_31, m2218_30 -# 2218| r2218_51(ClassWithDestructor) = Load[?] : &:r2218_48, ~m2218_50 -# 2218| m2218_52(ClassWithDestructor) = Store[y] : &:r2218_45, r2218_51 +# 2218| r2218_47(ClassWithDestructor) = Load[?] : &:r2218_46, ~m2218_38 +# 2218| m2218_48(ClassWithDestructor) = Store[y] : &:r2218_43, r2218_47 # 2219| r2219_1(glval) = VariableAddress[y] : # 2219| r2219_2(glval) = FunctionAddress[set_x] : # 2219| r2219_3(char) = Constant[97] : # 2219| v2219_4(void) = Call[set_x] : func:r2219_2, this:r2219_1, 0:r2219_3 -# 2219| m2219_5(unknown) = ^CallSideEffect : ~m2218_50 -# 2219| m2219_6(unknown) = Chi : total:m2218_50, partial:m2219_5 -# 2219| v2219_7(void) = ^IndirectReadSideEffect[-1] : &:r2219_1, m2218_52 +# 2219| m2219_5(unknown) = ^CallSideEffect : ~m2218_38 +# 2219| m2219_6(unknown) = Chi : total:m2218_38, partial:m2219_5 +# 2219| v2219_7(void) = ^IndirectReadSideEffect[-1] : &:r2219_1, m2218_48 # 2219| m2219_8(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2219_1 -# 2219| m2219_9(ClassWithDestructor) = Chi : total:m2218_52, partial:m2219_8 +# 2219| m2219_9(ClassWithDestructor) = Chi : total:m2218_48, partial:m2219_8 # 2220| r2220_1(glval) = VariableAddress[y] : # 2220| r2220_2(glval) = FunctionAddress[get_x] : # 2220| r2220_3(char) = Call[get_x] : func:r2220_2, this:r2220_1 @@ -13722,50 +13699,48 @@ ir.cpp: # 2221| Block 13 # 2221| v2221_1(void) = NoOp : -# 2218| r2218_53(glval) = VariableAddress[y] : -# 2218| r2218_54(glval) = FunctionAddress[~ClassWithDestructor] : -# 2218| v2218_55(void) = Call[~ClassWithDestructor] : func:r2218_54, this:r2218_53 -# 2218| m2218_56(unknown) = ^CallSideEffect : ~m2220_5 -# 2218| m2218_57(unknown) = Chi : total:m2220_5, partial:m2218_56 -# 2218| v2218_58(void) = ^IndirectReadSideEffect[-1] : &:r2218_53, m2220_8 -# 2218| m2218_59(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2218_53 -# 2218| m2218_60(ClassWithDestructor) = Chi : total:m2220_8, partial:m2218_59 -# 2218| r2218_61(glval>) = VariableAddress[ys] : -# 2218| r2218_62(glval) = FunctionAddress[~vector] : -# 2218| v2218_63(void) = Call[~vector] : func:r2218_62, this:r2218_61 -# 2218| m2218_64(unknown) = ^CallSideEffect : ~m2218_57 -# 2218| m2218_65(unknown) = Chi : total:m2218_57, partial:m2218_64 -# 2218| v2218_66(void) = ^IndirectReadSideEffect[-1] : &:r2218_61, m2218_13 -# 2218| m2218_67(vector) = ^IndirectMayWriteSideEffect[-1] : &:r2218_61 -# 2218| m2218_68(vector) = Chi : total:m2218_13, partial:m2218_67 +# 2218| r2218_49(glval) = VariableAddress[y] : +# 2218| r2218_50(glval) = FunctionAddress[~ClassWithDestructor] : +# 2218| v2218_51(void) = Call[~ClassWithDestructor] : func:r2218_50, this:r2218_49 +# 2218| m2218_52(unknown) = ^CallSideEffect : ~m2220_5 +# 2218| m2218_53(unknown) = Chi : total:m2220_5, partial:m2218_52 +# 2218| v2218_54(void) = ^IndirectReadSideEffect[-1] : &:r2218_49, m2220_8 +# 2218| m2218_55(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2218_49 +# 2218| m2218_56(ClassWithDestructor) = Chi : total:m2220_8, partial:m2218_55 +# 2218| r2218_57(glval>) = VariableAddress[ys] : +# 2218| r2218_58(glval) = FunctionAddress[~vector] : +# 2218| v2218_59(void) = Call[~vector] : func:r2218_58, this:r2218_57 +# 2218| m2218_60(unknown) = ^CallSideEffect : ~m2218_53 +# 2218| m2218_61(unknown) = Chi : total:m2218_53, partial:m2218_60 +# 2218| v2218_62(void) = ^IndirectReadSideEffect[-1] : &:r2218_57, m2218_13 +# 2218| m2218_63(vector) = ^IndirectMayWriteSideEffect[-1] : &:r2218_57 +# 2218| m2218_64(vector) = Chi : total:m2218_13, partial:m2218_63 # 2233| r2233_1(glval) = VariableAddress[x] : # 2233| r2233_2(glval) = FunctionAddress[~ClassWithDestructor] : # 2233| v2233_3(void) = Call[~ClassWithDestructor] : func:r2233_2, this:r2233_1 -# 2233| m2233_4(unknown) = ^CallSideEffect : ~m2218_65 -# 2233| m2233_5(unknown) = Chi : total:m2218_65, partial:m2233_4 +# 2233| m2233_4(unknown) = ^CallSideEffect : ~m2218_61 +# 2233| m2233_5(unknown) = Chi : total:m2218_61, partial:m2233_4 # 2233| v2233_6(void) = ^IndirectReadSideEffect[-1] : &:r2233_1, m2214_8 # 2233| m2233_7(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2233_1 # 2233| m2233_8(ClassWithDestructor) = Chi : total:m2214_8, partial:m2233_7 #-----| Goto -> Block 1 # 2218| Block 14 -# 2218| r2218_69(glval) = VariableAddress[y] : -# 2218| r2218_70(glval) = FunctionAddress[~ClassWithDestructor] : -# 2218| v2218_71(void) = Call[~ClassWithDestructor] : func:r2218_70, this:r2218_69 -# 2218| m2218_72(unknown) = ^CallSideEffect : ~m2220_5 -# 2218| m2218_73(unknown) = Chi : total:m2220_5, partial:m2218_72 -# 2218| v2218_74(void) = ^IndirectReadSideEffect[-1] : &:r2218_69, m2220_8 -# 2218| m2218_75(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2218_69 -# 2218| m2218_76(ClassWithDestructor) = Chi : total:m2220_8, partial:m2218_75 -# 2218| r2218_77(glval>) = VariableAddress[(__begin)] : -# 2218| r2218_78(glval) = FunctionAddress[operator++] : -# 2218| r2218_79(iterator &) = Call[operator++] : func:r2218_78, this:r2218_77 -# 2218| m2218_80(unknown) = ^CallSideEffect : ~m2218_73 -# 2218| m2218_81(unknown) = Chi : total:m2218_73, partial:m2218_80 -# 2218| v2218_82(void) = ^IndirectReadSideEffect[-1] : &:r2218_77, m2218_30 -# 2218| m2218_83(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2218_77 -# 2218| m2218_84(iterator) = Chi : total:m2218_30, partial:m2218_83 -# 2218| r2218_85(glval>) = CopyValue : r2218_79 +# 2218| r2218_65(glval) = VariableAddress[y] : +# 2218| r2218_66(glval) = FunctionAddress[~ClassWithDestructor] : +# 2218| v2218_67(void) = Call[~ClassWithDestructor] : func:r2218_66, this:r2218_65 +# 2218| m2218_68(unknown) = ^CallSideEffect : ~m2220_5 +# 2218| m2218_69(unknown) = Chi : total:m2220_5, partial:m2218_68 +# 2218| v2218_70(void) = ^IndirectReadSideEffect[-1] : &:r2218_65, m2220_8 +# 2218| m2218_71(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2218_65 +# 2218| m2218_72(ClassWithDestructor) = Chi : total:m2220_8, partial:m2218_71 +# 2218| r2218_73(glval>) = VariableAddress[(__begin)] : +# 2218| r2218_74(glval) = FunctionAddress[operator++] : +# 2218| r2218_75(iterator &) = Call[operator++] : func:r2218_74, this:r2218_73 +# 2218| v2218_76(void) = ^IndirectReadSideEffect[-1] : &:r2218_73, m2218_30 +# 2218| m2218_77(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2218_73 +# 2218| m2218_78(iterator) = Chi : total:m2218_30, partial:m2218_77 +# 2218| r2218_79(glval>) = CopyValue : r2218_75 #-----| Goto (back edge) -> Block 11 # 2224| Block 15 @@ -13774,8 +13749,8 @@ ir.cpp: # 2224| r2224_3(glval) = FunctionAddress[vector] : # 2224| r2224_4(int) = Constant[1] : # 2224| v2224_5(void) = Call[vector] : func:r2224_3, this:r2224_1, 0:r2224_4 -# 2224| m2224_6(unknown) = ^CallSideEffect : ~m2218_43 -# 2224| m2224_7(unknown) = Chi : total:m2218_43, partial:m2224_6 +# 2224| m2224_6(unknown) = ^CallSideEffect : ~m2218_38 +# 2224| m2224_7(unknown) = Chi : total:m2218_38, partial:m2224_6 # 2224| m2224_8(vector) = ^IndirectMayWriteSideEffect[-1] : &:r2224_1 # 2224| m2224_9(vector) = Chi : total:m2224_2, partial:m2224_8 # 2224| r2224_10(glval &>) = VariableAddress[(__range)] : @@ -13803,8 +13778,8 @@ ir.cpp: #-----| Goto -> Block 16 # 2224| Block 16 -# 2224| m2224_26(iterator) = Phi : from 15:m2224_19, from 17:m2224_48 -# 2224| m2224_27(unknown) = Phi : from 15:~m2224_7, from 17:~m2224_45 +# 2224| m2224_26(iterator) = Phi : from 15:m2224_19, from 17:m2224_44 +# 2224| m2224_27(unknown) = Phi : from 15:~m2224_7, from 17:~m2224_34 # 2224| r2224_28(glval>) = VariableAddress[(__begin)] : #-----| r0_39(glval>) = Convert : r2224_28 # 2224| r2224_29(glval) = FunctionAddress[operator!=] : @@ -13822,38 +13797,32 @@ ir.cpp: # 2224| m2224_36(iterator) = Chi : total:m0_41, partial:m2224_35 #-----| r0_45(iterator) = Load[#temp0:0] : &:r0_40, m2224_36 # 2224| r2224_37(bool) = Call[operator!=] : func:r2224_29, this:r0_39, 0:r0_45 -# 2224| m2224_38(unknown) = ^CallSideEffect : ~m2224_34 -# 2224| m2224_39(unknown) = Chi : total:m2224_34, partial:m2224_38 #-----| v0_46(void) = ^IndirectReadSideEffect[-1] : &:r0_39, m2224_26 -# 2224| v2224_40(void) = ConditionalBranch : r2224_37 +# 2224| v2224_38(void) = ConditionalBranch : r2224_37 #-----| False -> Block 20 #-----| True -> Block 18 # 2224| Block 17 -# 2224| r2224_41(glval>) = VariableAddress[(__begin)] : -# 2224| r2224_42(glval) = FunctionAddress[operator++] : -# 2224| r2224_43(iterator &) = Call[operator++] : func:r2224_42, this:r2224_41 -# 2224| m2224_44(unknown) = ^CallSideEffect : ~m2224_55 -# 2224| m2224_45(unknown) = Chi : total:m2224_55, partial:m2224_44 -# 2224| v2224_46(void) = ^IndirectReadSideEffect[-1] : &:r2224_41, m2224_26 -# 2224| m2224_47(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2224_41 -# 2224| m2224_48(iterator) = Chi : total:m2224_26, partial:m2224_47 -# 2224| r2224_49(glval>) = CopyValue : r2224_43 +# 2224| r2224_39(glval>) = VariableAddress[(__begin)] : +# 2224| r2224_40(glval) = FunctionAddress[operator++] : +# 2224| r2224_41(iterator &) = Call[operator++] : func:r2224_40, this:r2224_39 +# 2224| v2224_42(void) = ^IndirectReadSideEffect[-1] : &:r2224_39, m2224_26 +# 2224| m2224_43(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2224_39 +# 2224| m2224_44(iterator) = Chi : total:m2224_26, partial:m2224_43 +# 2224| r2224_45(glval>) = CopyValue : r2224_41 #-----| Goto (back edge) -> Block 16 # 2224| Block 18 -# 2224| r2224_50(glval) = VariableAddress[y] : -# 2224| r2224_51(glval>) = VariableAddress[(__begin)] : -#-----| r0_47(glval>) = Convert : r2224_51 -# 2224| r2224_52(glval) = FunctionAddress[operator*] : -# 2224| r2224_53(int &) = Call[operator*] : func:r2224_52, this:r0_47 -# 2224| m2224_54(unknown) = ^CallSideEffect : ~m2224_39 -# 2224| m2224_55(unknown) = Chi : total:m2224_39, partial:m2224_54 +# 2224| r2224_46(glval) = VariableAddress[y] : +# 2224| r2224_47(glval>) = VariableAddress[(__begin)] : +#-----| r0_47(glval>) = Convert : r2224_47 +# 2224| r2224_48(glval) = FunctionAddress[operator*] : +# 2224| r2224_49(int &) = Call[operator*] : func:r2224_48, this:r0_47 #-----| v0_48(void) = ^IndirectReadSideEffect[-1] : &:r0_47, m2224_26 -# 2224| r2224_56(int) = Load[?] : &:r2224_53, ~m2224_55 -# 2224| m2224_57(int) = Store[y] : &:r2224_50, r2224_56 +# 2224| r2224_50(int) = Load[?] : &:r2224_49, ~m2224_34 +# 2224| m2224_51(int) = Store[y] : &:r2224_46, r2224_50 # 2225| r2225_1(glval) = VariableAddress[y] : -# 2225| r2225_2(int) = Load[y] : &:r2225_1, m2224_57 +# 2225| r2225_2(int) = Load[y] : &:r2225_1, m2224_51 # 2225| r2225_3(int) = Constant[1] : # 2225| r2225_4(bool) = CompareEQ : r2225_2, r2225_3 # 2225| v2225_5(void) = ConditionalBranch : r2225_4 @@ -13862,19 +13831,19 @@ ir.cpp: # 2226| Block 19 # 2226| v2226_1(void) = NoOp : -# 2224| r2224_58(glval>) = VariableAddress[ys] : -# 2224| r2224_59(glval) = FunctionAddress[~vector] : -# 2224| v2224_60(void) = Call[~vector] : func:r2224_59, this:r2224_58 -# 2224| m2224_61(unknown) = ^CallSideEffect : ~m2224_55 -# 2224| m2224_62(unknown) = Chi : total:m2224_55, partial:m2224_61 -# 2224| v2224_63(void) = ^IndirectReadSideEffect[-1] : &:r2224_58, m2224_9 -# 2224| m2224_64(vector) = ^IndirectMayWriteSideEffect[-1] : &:r2224_58 -# 2224| m2224_65(vector) = Chi : total:m2224_9, partial:m2224_64 +# 2224| r2224_52(glval>) = VariableAddress[ys] : +# 2224| r2224_53(glval) = FunctionAddress[~vector] : +# 2224| v2224_54(void) = Call[~vector] : func:r2224_53, this:r2224_52 +# 2224| m2224_55(unknown) = ^CallSideEffect : ~m2224_34 +# 2224| m2224_56(unknown) = Chi : total:m2224_34, partial:m2224_55 +# 2224| v2224_57(void) = ^IndirectReadSideEffect[-1] : &:r2224_52, m2224_9 +# 2224| m2224_58(vector) = ^IndirectMayWriteSideEffect[-1] : &:r2224_52 +# 2224| m2224_59(vector) = Chi : total:m2224_9, partial:m2224_58 # 2233| r2233_9(glval) = VariableAddress[x] : # 2233| r2233_10(glval) = FunctionAddress[~ClassWithDestructor] : # 2233| v2233_11(void) = Call[~ClassWithDestructor] : func:r2233_10, this:r2233_9 -# 2233| m2233_12(unknown) = ^CallSideEffect : ~m2224_62 -# 2233| m2233_13(unknown) = Chi : total:m2224_62, partial:m2233_12 +# 2233| m2233_12(unknown) = ^CallSideEffect : ~m2224_56 +# 2233| m2233_13(unknown) = Chi : total:m2224_56, partial:m2233_12 # 2233| v2233_14(void) = ^IndirectReadSideEffect[-1] : &:r2233_9, m2214_8 # 2233| m2233_15(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2233_9 # 2233| m2233_16(ClassWithDestructor) = Chi : total:m2214_8, partial:m2233_15 @@ -13890,8 +13859,8 @@ ir.cpp: # 2229| m2229_7(ClassWithDestructor) = Store[#temp2229:45] : &:r2229_4, r2229_6 # 2229| r2229_8(ClassWithDestructor) = Load[#temp2229:45] : &:r2229_4, m2229_7 # 2229| v2229_9(void) = Call[vector] : func:r2229_3, this:r2229_1, 0:r2229_8 -# 2229| m2229_10(unknown) = ^CallSideEffect : ~m2224_39 -# 2229| m2229_11(unknown) = Chi : total:m2224_39, partial:m2229_10 +# 2229| m2229_10(unknown) = ^CallSideEffect : ~m2224_34 +# 2229| m2229_11(unknown) = Chi : total:m2224_34, partial:m2229_10 # 2229| m2229_12(vector) = ^IndirectMayWriteSideEffect[-1] : &:r2229_1 # 2229| m2229_13(vector) = Chi : total:m2229_2, partial:m2229_12 # 2229| r2229_14(glval &>) = VariableAddress[(__range)] : @@ -13919,8 +13888,8 @@ ir.cpp: #-----| Goto -> Block 21 # 2229| Block 21 -# 2229| m2229_30(iterator) = Phi : from 20:m2229_23, from 22:m2229_68 -# 2229| m2229_31(unknown) = Phi : from 20:~m2229_11, from 22:~m2229_65 +# 2229| m2229_30(iterator) = Phi : from 20:m2229_23, from 22:m2229_62 +# 2229| m2229_31(unknown) = Phi : from 20:~m2229_11, from 22:~m2229_53 # 2229| r2229_32(glval>) = VariableAddress[(__begin)] : #-----| r0_55(glval>) = Convert : r2229_32 # 2229| r2229_33(glval) = FunctionAddress[operator!=] : @@ -13938,30 +13907,26 @@ ir.cpp: # 2229| m2229_40(iterator) = Chi : total:m0_57, partial:m2229_39 #-----| r0_61(iterator) = Load[#temp0:0] : &:r0_56, m2229_40 # 2229| r2229_41(bool) = Call[operator!=] : func:r2229_33, this:r0_55, 0:r0_61 -# 2229| m2229_42(unknown) = ^CallSideEffect : ~m2229_38 -# 2229| m2229_43(unknown) = Chi : total:m2229_38, partial:m2229_42 #-----| v0_62(void) = ^IndirectReadSideEffect[-1] : &:r0_55, m2229_30 -# 2229| v2229_44(void) = ConditionalBranch : r2229_41 +# 2229| v2229_42(void) = ConditionalBranch : r2229_41 #-----| False -> Block 23 #-----| True -> Block 22 # 2229| Block 22 -# 2229| r2229_45(glval) = VariableAddress[y] : -# 2229| r2229_46(glval>) = VariableAddress[(__begin)] : -#-----| r0_63(glval>) = Convert : r2229_46 -# 2229| r2229_47(glval) = FunctionAddress[operator*] : -# 2229| r2229_48(ClassWithDestructor &) = Call[operator*] : func:r2229_47, this:r0_63 -# 2229| m2229_49(unknown) = ^CallSideEffect : ~m2229_43 -# 2229| m2229_50(unknown) = Chi : total:m2229_43, partial:m2229_49 +# 2229| r2229_43(glval) = VariableAddress[y] : +# 2229| r2229_44(glval>) = VariableAddress[(__begin)] : +#-----| r0_63(glval>) = Convert : r2229_44 +# 2229| r2229_45(glval) = FunctionAddress[operator*] : +# 2229| r2229_46(ClassWithDestructor &) = Call[operator*] : func:r2229_45, this:r0_63 #-----| v0_64(void) = ^IndirectReadSideEffect[-1] : &:r0_63, m2229_30 -# 2229| r2229_51(ClassWithDestructor) = Load[?] : &:r2229_48, ~m2229_50 -# 2229| m2229_52(ClassWithDestructor) = Store[y] : &:r2229_45, r2229_51 +# 2229| r2229_47(ClassWithDestructor) = Load[?] : &:r2229_46, ~m2229_38 +# 2229| m2229_48(ClassWithDestructor) = Store[y] : &:r2229_43, r2229_47 # 2230| r2230_1(glval) = VariableAddress[z1] : # 2230| m2230_2(ClassWithDestructor) = Uninitialized[z1] : &:r2230_1 # 2230| r2230_3(glval) = FunctionAddress[ClassWithDestructor] : # 2230| v2230_4(void) = Call[ClassWithDestructor] : func:r2230_3, this:r2230_1 -# 2230| m2230_5(unknown) = ^CallSideEffect : ~m2229_50 -# 2230| m2230_6(unknown) = Chi : total:m2229_50, partial:m2230_5 +# 2230| m2230_5(unknown) = ^CallSideEffect : ~m2229_38 +# 2230| m2230_6(unknown) = Chi : total:m2229_38, partial:m2230_5 # 2230| m2230_7(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2230_1 # 2230| m2230_8(ClassWithDestructor) = Chi : total:m2230_2, partial:m2230_7 # 2231| r2231_1(glval) = VariableAddress[z2] : @@ -13988,23 +13953,21 @@ ir.cpp: # 2232| v2232_14(void) = ^IndirectReadSideEffect[-1] : &:r2232_9, m2230_8 # 2232| m2232_15(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2232_9 # 2232| m2232_16(ClassWithDestructor) = Chi : total:m2230_8, partial:m2232_15 -# 2229| r2229_53(glval) = VariableAddress[y] : -# 2229| r2229_54(glval) = FunctionAddress[~ClassWithDestructor] : -# 2229| v2229_55(void) = Call[~ClassWithDestructor] : func:r2229_54, this:r2229_53 -# 2229| m2229_56(unknown) = ^CallSideEffect : ~m2232_13 -# 2229| m2229_57(unknown) = Chi : total:m2232_13, partial:m2229_56 -# 2229| v2229_58(void) = ^IndirectReadSideEffect[-1] : &:r2229_53, m2229_52 -# 2229| m2229_59(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2229_53 -# 2229| m2229_60(ClassWithDestructor) = Chi : total:m2229_52, partial:m2229_59 -# 2229| r2229_61(glval>) = VariableAddress[(__begin)] : -# 2229| r2229_62(glval) = FunctionAddress[operator++] : -# 2229| r2229_63(iterator &) = Call[operator++] : func:r2229_62, this:r2229_61 -# 2229| m2229_64(unknown) = ^CallSideEffect : ~m2229_57 -# 2229| m2229_65(unknown) = Chi : total:m2229_57, partial:m2229_64 -# 2229| v2229_66(void) = ^IndirectReadSideEffect[-1] : &:r2229_61, m2229_30 -# 2229| m2229_67(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2229_61 -# 2229| m2229_68(iterator) = Chi : total:m2229_30, partial:m2229_67 -# 2229| r2229_69(glval>) = CopyValue : r2229_63 +# 2229| r2229_49(glval) = VariableAddress[y] : +# 2229| r2229_50(glval) = FunctionAddress[~ClassWithDestructor] : +# 2229| v2229_51(void) = Call[~ClassWithDestructor] : func:r2229_50, this:r2229_49 +# 2229| m2229_52(unknown) = ^CallSideEffect : ~m2232_13 +# 2229| m2229_53(unknown) = Chi : total:m2232_13, partial:m2229_52 +# 2229| v2229_54(void) = ^IndirectReadSideEffect[-1] : &:r2229_49, m2229_48 +# 2229| m2229_55(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2229_49 +# 2229| m2229_56(ClassWithDestructor) = Chi : total:m2229_48, partial:m2229_55 +# 2229| r2229_57(glval>) = VariableAddress[(__begin)] : +# 2229| r2229_58(glval) = FunctionAddress[operator++] : +# 2229| r2229_59(iterator &) = Call[operator++] : func:r2229_58, this:r2229_57 +# 2229| v2229_60(void) = ^IndirectReadSideEffect[-1] : &:r2229_57, m2229_30 +# 2229| m2229_61(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2229_57 +# 2229| m2229_62(iterator) = Chi : total:m2229_30, partial:m2229_61 +# 2229| r2229_63(glval>) = CopyValue : r2229_59 #-----| Goto (back edge) -> Block 21 # 2233| Block 23 @@ -14012,8 +13975,8 @@ ir.cpp: # 2233| r2233_18(glval) = VariableAddress[x] : # 2233| r2233_19(glval) = FunctionAddress[~ClassWithDestructor] : # 2233| v2233_20(void) = Call[~ClassWithDestructor] : func:r2233_19, this:r2233_18 -# 2233| m2233_21(unknown) = ^CallSideEffect : ~m2229_43 -# 2233| m2233_22(unknown) = Chi : total:m2229_43, partial:m2233_21 +# 2233| m2233_21(unknown) = ^CallSideEffect : ~m2229_38 +# 2233| m2233_22(unknown) = Chi : total:m2229_38, partial:m2233_21 # 2233| v2233_23(void) = ^IndirectReadSideEffect[-1] : &:r2233_18, m2214_8 # 2233| m2233_24(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2233_18 # 2233| m2233_25(ClassWithDestructor) = Chi : total:m2214_8, partial:m2233_24 @@ -14685,8 +14648,8 @@ ir.cpp: #-----| Goto -> Block 4 # 2307| Block 4 -# 2307| m2307_36(iterator) = Phi : from 3:m2307_29, from 5:m2307_83 -# 2307| m2307_37(unknown) = Phi : from 3:~m2307_19, from 5:~m2307_80 +# 2307| m2307_36(iterator) = Phi : from 3:m2307_29, from 5:m2307_77 +# 2307| m2307_37(unknown) = Phi : from 3:~m2307_19, from 5:~m2307_68 # 2307| r2307_38(glval>) = VariableAddress[(__begin)] : #-----| r0_7(glval>) = Convert : r2307_38 # 2307| r2307_39(glval) = FunctionAddress[operator!=] : @@ -14704,39 +14667,35 @@ ir.cpp: # 2307| m2307_46(iterator) = Chi : total:m0_9, partial:m2307_45 #-----| r0_13(iterator) = Load[#temp0:0] : &:r0_8, m2307_46 # 2307| r2307_47(bool) = Call[operator!=] : func:r2307_39, this:r0_7, 0:r0_13 -# 2307| m2307_48(unknown) = ^CallSideEffect : ~m2307_44 -# 2307| m2307_49(unknown) = Chi : total:m2307_44, partial:m2307_48 #-----| v0_14(void) = ^IndirectReadSideEffect[-1] : &:r0_7, m2307_36 -# 2307| v2307_50(void) = ConditionalBranch : r2307_47 +# 2307| v2307_48(void) = ConditionalBranch : r2307_47 #-----| False -> Block 6 #-----| True -> Block 5 # 2307| Block 5 -# 2307| r2307_51(glval) = VariableAddress[s] : -# 2307| m2307_52(String) = Uninitialized[s] : &:r2307_51 -# 2307| r2307_53(glval) = FunctionAddress[String] : -# 2307| r2307_54(glval>) = VariableAddress[(__begin)] : -#-----| r0_15(glval>) = Convert : r2307_54 -# 2307| r2307_55(glval) = FunctionAddress[operator*] : -# 2307| r2307_56(String &) = Call[operator*] : func:r2307_55, this:r0_15 -# 2307| m2307_57(unknown) = ^CallSideEffect : ~m2307_49 -# 2307| m2307_58(unknown) = Chi : total:m2307_49, partial:m2307_57 +# 2307| r2307_49(glval) = VariableAddress[s] : +# 2307| m2307_50(String) = Uninitialized[s] : &:r2307_49 +# 2307| r2307_51(glval) = FunctionAddress[String] : +# 2307| r2307_52(glval>) = VariableAddress[(__begin)] : +#-----| r0_15(glval>) = Convert : r2307_52 +# 2307| r2307_53(glval) = FunctionAddress[operator*] : +# 2307| r2307_54(String &) = Call[operator*] : func:r2307_53, this:r0_15 #-----| v0_16(void) = ^IndirectReadSideEffect[-1] : &:r0_15, m2307_36 -# 2307| r2307_59(glval) = CopyValue : r2307_56 -# 2307| r2307_60(glval) = Convert : r2307_59 -# 2307| r2307_61(String &) = CopyValue : r2307_60 -# 2307| v2307_62(void) = Call[String] : func:r2307_53, this:r2307_51, 0:r2307_61 -# 2307| m2307_63(unknown) = ^CallSideEffect : ~m2307_58 -# 2307| m2307_64(unknown) = Chi : total:m2307_58, partial:m2307_63 -# 2307| v2307_65(void) = ^BufferReadSideEffect[0] : &:r2307_61, ~m2307_64 -# 2307| m2307_66(String) = ^IndirectMayWriteSideEffect[-1] : &:r2307_51 -# 2307| m2307_67(String) = Chi : total:m2307_52, partial:m2307_66 +# 2307| r2307_55(glval) = CopyValue : r2307_54 +# 2307| r2307_56(glval) = Convert : r2307_55 +# 2307| r2307_57(String &) = CopyValue : r2307_56 +# 2307| v2307_58(void) = Call[String] : func:r2307_51, this:r2307_49, 0:r2307_57 +# 2307| m2307_59(unknown) = ^CallSideEffect : ~m2307_44 +# 2307| m2307_60(unknown) = Chi : total:m2307_44, partial:m2307_59 +# 2307| v2307_61(void) = ^BufferReadSideEffect[0] : &:r2307_57, ~m2307_60 +# 2307| m2307_62(String) = ^IndirectMayWriteSideEffect[-1] : &:r2307_49 +# 2307| m2307_63(String) = Chi : total:m2307_50, partial:m2307_62 # 2308| r2308_1(glval) = VariableAddress[s2] : # 2308| m2308_2(String) = Uninitialized[s2] : &:r2308_1 # 2308| r2308_3(glval) = FunctionAddress[String] : # 2308| v2308_4(void) = Call[String] : func:r2308_3, this:r2308_1 -# 2308| m2308_5(unknown) = ^CallSideEffect : ~m2307_64 -# 2308| m2308_6(unknown) = Chi : total:m2307_64, partial:m2308_5 +# 2308| m2308_5(unknown) = ^CallSideEffect : ~m2307_60 +# 2308| m2308_6(unknown) = Chi : total:m2307_60, partial:m2308_5 # 2308| m2308_7(String) = ^IndirectMayWriteSideEffect[-1] : &:r2308_1 # 2308| m2308_8(String) = Chi : total:m2308_2, partial:m2308_7 # 2309| r2309_1(glval) = VariableAddress[s2] : @@ -14747,23 +14706,21 @@ ir.cpp: # 2309| v2309_6(void) = ^IndirectReadSideEffect[-1] : &:r2309_1, m2308_8 # 2309| m2309_7(String) = ^IndirectMayWriteSideEffect[-1] : &:r2309_1 # 2309| m2309_8(String) = Chi : total:m2308_8, partial:m2309_7 -# 2307| r2307_68(glval) = VariableAddress[s] : -# 2307| r2307_69(glval) = FunctionAddress[~String] : -# 2307| v2307_70(void) = Call[~String] : func:r2307_69, this:r2307_68 -# 2307| m2307_71(unknown) = ^CallSideEffect : ~m2309_5 -# 2307| m2307_72(unknown) = Chi : total:m2309_5, partial:m2307_71 -# 2307| v2307_73(void) = ^IndirectReadSideEffect[-1] : &:r2307_68, m2307_67 -# 2307| m2307_74(String) = ^IndirectMayWriteSideEffect[-1] : &:r2307_68 -# 2307| m2307_75(String) = Chi : total:m2307_67, partial:m2307_74 -# 2307| r2307_76(glval>) = VariableAddress[(__begin)] : -# 2307| r2307_77(glval) = FunctionAddress[operator++] : -# 2307| r2307_78(iterator &) = Call[operator++] : func:r2307_77, this:r2307_76 -# 2307| m2307_79(unknown) = ^CallSideEffect : ~m2307_72 -# 2307| m2307_80(unknown) = Chi : total:m2307_72, partial:m2307_79 -# 2307| v2307_81(void) = ^IndirectReadSideEffect[-1] : &:r2307_76, m2307_36 -# 2307| m2307_82(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2307_76 -# 2307| m2307_83(iterator) = Chi : total:m2307_36, partial:m2307_82 -# 2307| r2307_84(glval>) = CopyValue : r2307_78 +# 2307| r2307_64(glval) = VariableAddress[s] : +# 2307| r2307_65(glval) = FunctionAddress[~String] : +# 2307| v2307_66(void) = Call[~String] : func:r2307_65, this:r2307_64 +# 2307| m2307_67(unknown) = ^CallSideEffect : ~m2309_5 +# 2307| m2307_68(unknown) = Chi : total:m2309_5, partial:m2307_67 +# 2307| v2307_69(void) = ^IndirectReadSideEffect[-1] : &:r2307_64, m2307_63 +# 2307| m2307_70(String) = ^IndirectMayWriteSideEffect[-1] : &:r2307_64 +# 2307| m2307_71(String) = Chi : total:m2307_63, partial:m2307_70 +# 2307| r2307_72(glval>) = VariableAddress[(__begin)] : +# 2307| r2307_73(glval) = FunctionAddress[operator++] : +# 2307| r2307_74(iterator &) = Call[operator++] : func:r2307_73, this:r2307_72 +# 2307| v2307_75(void) = ^IndirectReadSideEffect[-1] : &:r2307_72, m2307_36 +# 2307| m2307_76(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2307_72 +# 2307| m2307_77(iterator) = Chi : total:m2307_36, partial:m2307_76 +# 2307| r2307_78(glval>) = CopyValue : r2307_74 #-----| Goto (back edge) -> Block 4 # 2311| Block 6 @@ -14773,8 +14730,8 @@ ir.cpp: # 2311| r2311_4(glval) = StringConstant["hello"] : # 2311| r2311_5(char *) = Convert : r2311_4 # 2311| v2311_6(void) = Call[String] : func:r2311_3, this:r2311_1, 0:r2311_5 -# 2311| m2311_7(unknown) = ^CallSideEffect : ~m2307_49 -# 2311| m2311_8(unknown) = Chi : total:m2307_49, partial:m2311_7 +# 2311| m2311_7(unknown) = ^CallSideEffect : ~m2307_44 +# 2311| m2311_8(unknown) = Chi : total:m2307_44, partial:m2311_7 # 2311| v2311_9(void) = ^BufferReadSideEffect[0] : &:r2311_5, ~m2301_3 # 2311| m2311_10(String) = ^IndirectMayWriteSideEffect[-1] : &:r2311_1 # 2311| m2311_11(String) = Chi : total:m2311_2, partial:m2311_10 @@ -15517,8 +15474,8 @@ ir.cpp: #-----| Goto -> Block 10 # 2430| Block 10 -# 2430| m2430_43(iterator) = Phi : from 9:m2430_36, from 11:m2430_73 -# 2430| m2430_44(unknown) = Phi : from 9:~m2430_26, from 11:~m2430_70 +# 2430| m2430_43(iterator) = Phi : from 9:m2430_36, from 11:m2430_67 +# 2430| m2430_44(unknown) = Phi : from 9:~m2430_26, from 11:~m2430_51 # 2430| r2430_45(glval>) = VariableAddress[(__begin)] : #-----| r0_7(glval>) = Convert : r2430_45 # 2430| r2430_46(glval) = FunctionAddress[operator!=] : @@ -15536,48 +15493,42 @@ ir.cpp: # 2430| m2430_53(iterator) = Chi : total:m0_9, partial:m2430_52 #-----| r0_13(iterator) = Load[#temp0:0] : &:r0_8, m2430_53 # 2430| r2430_54(bool) = Call[operator!=] : func:r2430_46, this:r0_7, 0:r0_13 -# 2430| m2430_55(unknown) = ^CallSideEffect : ~m2430_51 -# 2430| m2430_56(unknown) = Chi : total:m2430_51, partial:m2430_55 #-----| v0_14(void) = ^IndirectReadSideEffect[-1] : &:r0_7, m2430_43 -# 2430| v2430_57(void) = ConditionalBranch : r2430_54 +# 2430| v2430_55(void) = ConditionalBranch : r2430_54 #-----| False -> Block 12 #-----| True -> Block 11 # 2430| Block 11 -# 2430| r2430_58(glval) = VariableAddress[y] : -# 2430| r2430_59(glval>) = VariableAddress[(__begin)] : -#-----| r0_15(glval>) = Convert : r2430_59 -# 2430| r2430_60(glval) = FunctionAddress[operator*] : -# 2430| r2430_61(char &) = Call[operator*] : func:r2430_60, this:r0_15 -# 2430| m2430_62(unknown) = ^CallSideEffect : ~m2430_56 -# 2430| m2430_63(unknown) = Chi : total:m2430_56, partial:m2430_62 +# 2430| r2430_56(glval) = VariableAddress[y] : +# 2430| r2430_57(glval>) = VariableAddress[(__begin)] : +#-----| r0_15(glval>) = Convert : r2430_57 +# 2430| r2430_58(glval) = FunctionAddress[operator*] : +# 2430| r2430_59(char &) = Call[operator*] : func:r2430_58, this:r0_15 #-----| v0_16(void) = ^IndirectReadSideEffect[-1] : &:r0_15, m2430_43 -# 2430| r2430_64(char) = Load[?] : &:r2430_61, ~m2430_63 -# 2430| m2430_65(char) = Store[y] : &:r2430_58, r2430_64 +# 2430| r2430_60(char) = Load[?] : &:r2430_59, ~m2430_51 +# 2430| m2430_61(char) = Store[y] : &:r2430_56, r2430_60 # 2431| r2431_1(glval) = VariableAddress[x] : # 2431| r2431_2(char) = Load[x] : &:r2431_1, m2430_17 # 2431| r2431_3(int) = Convert : r2431_2 # 2431| r2431_4(glval) = VariableAddress[y] : -# 2431| r2431_5(char) = Load[y] : &:r2431_4, m2430_65 +# 2431| r2431_5(char) = Load[y] : &:r2431_4, m2430_61 # 2431| r2431_6(int) = Convert : r2431_5 # 2431| r2431_7(int) = Add : r2431_6, r2431_3 # 2431| r2431_8(char) = Convert : r2431_7 # 2431| m2431_9(char) = Store[y] : &:r2431_4, r2431_8 -# 2430| r2430_66(glval>) = VariableAddress[(__begin)] : -# 2430| r2430_67(glval) = FunctionAddress[operator++] : -# 2430| r2430_68(iterator &) = Call[operator++] : func:r2430_67, this:r2430_66 -# 2430| m2430_69(unknown) = ^CallSideEffect : ~m2430_63 -# 2430| m2430_70(unknown) = Chi : total:m2430_63, partial:m2430_69 -# 2430| v2430_71(void) = ^IndirectReadSideEffect[-1] : &:r2430_66, m2430_43 -# 2430| m2430_72(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2430_66 -# 2430| m2430_73(iterator) = Chi : total:m2430_43, partial:m2430_72 -# 2430| r2430_74(glval>) = CopyValue : r2430_68 +# 2430| r2430_62(glval>) = VariableAddress[(__begin)] : +# 2430| r2430_63(glval) = FunctionAddress[operator++] : +# 2430| r2430_64(iterator &) = Call[operator++] : func:r2430_63, this:r2430_62 +# 2430| v2430_65(void) = ^IndirectReadSideEffect[-1] : &:r2430_62, m2430_43 +# 2430| m2430_66(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2430_62 +# 2430| m2430_67(iterator) = Chi : total:m2430_43, partial:m2430_66 +# 2430| r2430_68(glval>) = CopyValue : r2430_64 #-----| Goto (back edge) -> Block 10 # 2432| Block 12 # 2432| v2432_1(void) = NoOp : # 2410| v2410_5(void) = ReturnVoid : -# 2410| v2410_6(void) = AliasedUse : ~m2430_56 +# 2410| v2410_6(void) = AliasedUse : ~m2430_51 # 2410| v2410_7(void) = ExitFunction : # 2410| Block 13 diff --git a/cpp/ql/test/library-tests/ir/ir/operand_locations.expected b/cpp/ql/test/library-tests/ir/ir/operand_locations.expected index 2799fc28e94..17bbd5c7670 100644 --- a/cpp/ql/test/library-tests/ir/ir/operand_locations.expected +++ b/cpp/ql/test/library-tests/ir/ir/operand_locations.expected @@ -6293,7 +6293,7 @@ | ir.cpp:1056:1:1056:1 | SideEffect | ~m1056_6 | | ir.cpp:1126:6:1126:18 | ChiPartial | partial:m1126_3 | | ir.cpp:1126:6:1126:18 | ChiTotal | total:m1126_2 | -| ir.cpp:1126:6:1126:18 | SideEffect | ~m1137_1 | +| ir.cpp:1126:6:1126:18 | SideEffect | ~m1133_27 | | ir.cpp:1126:44:1126:44 | Address | &:r1126_5 | | ir.cpp:1126:44:1126:44 | Address | &:r1126_5 | | ir.cpp:1126:44:1126:44 | Address | &:r1126_7 | @@ -6303,50 +6303,41 @@ | ir.cpp:1127:5:1127:5 | Address | &:r1127_1 | | ir.cpp:1127:5:1127:5 | Address | &:r1127_7 | | ir.cpp:1127:5:1127:5 | Address | &:r1127_13 | -| ir.cpp:1127:14:1127:14 | Address | &:r1127_34 | +| ir.cpp:1127:14:1127:14 | Address | &:r1127_32 | | ir.cpp:1127:18:1127:18 | Address | &:r1127_2 | | ir.cpp:1127:18:1127:18 | Address | &:r1127_8 | | ir.cpp:1127:18:1127:18 | Address | &:r1127_14 | -| ir.cpp:1127:18:1127:18 | Address | &:r1127_37 | -| ir.cpp:1127:18:1127:18 | Address | &:r1127_43 | -| ir.cpp:1127:18:1127:18 | Address | &:r1127_43 | +| ir.cpp:1127:18:1127:18 | Address | &:r1127_35 | +| ir.cpp:1127:18:1127:18 | Address | &:r1127_39 | +| ir.cpp:1127:18:1127:18 | Address | &:r1127_39 | | ir.cpp:1127:18:1127:18 | Arg(this) | this:r0_1 | | ir.cpp:1127:18:1127:18 | Arg(this) | this:r0_3 | | ir.cpp:1127:18:1127:18 | Arg(this) | this:r0_5 | | ir.cpp:1127:18:1127:18 | Arg(this) | this:r0_6 | | ir.cpp:1127:18:1127:18 | Arg(this) | this:r0_13 | -| ir.cpp:1127:18:1127:18 | Arg(this) | this:r1127_43 | +| ir.cpp:1127:18:1127:18 | Arg(this) | this:r1127_39 | | ir.cpp:1127:18:1127:18 | CallTarget | func:r1127_10 | | ir.cpp:1127:18:1127:18 | CallTarget | func:r1127_16 | | ir.cpp:1127:18:1127:18 | CallTarget | func:r1127_22 | | ir.cpp:1127:18:1127:18 | CallTarget | func:r1127_23 | -| ir.cpp:1127:18:1127:18 | CallTarget | func:r1127_36 | -| ir.cpp:1127:18:1127:18 | CallTarget | func:r1127_44 | +| ir.cpp:1127:18:1127:18 | CallTarget | func:r1127_34 | +| ir.cpp:1127:18:1127:18 | CallTarget | func:r1127_40 | | ir.cpp:1127:18:1127:18 | ChiPartial | partial:m1127_26 | | ir.cpp:1127:18:1127:18 | ChiPartial | partial:m1127_28 | -| ir.cpp:1127:18:1127:18 | ChiPartial | partial:m1127_31 | -| ir.cpp:1127:18:1127:18 | ChiPartial | partial:m1127_38 | -| ir.cpp:1127:18:1127:18 | ChiPartial | partial:m1127_46 | -| ir.cpp:1127:18:1127:18 | ChiPartial | partial:m1127_49 | +| ir.cpp:1127:18:1127:18 | ChiPartial | partial:m1127_43 | | ir.cpp:1127:18:1127:18 | ChiTotal | total:m0_7 | | ir.cpp:1127:18:1127:18 | ChiTotal | total:m1127_19 | | ir.cpp:1127:18:1127:18 | ChiTotal | total:m1127_20 | -| ir.cpp:1127:18:1127:18 | ChiTotal | total:m1127_27 | -| ir.cpp:1127:18:1127:18 | ChiTotal | total:m1127_32 | -| ir.cpp:1127:18:1127:18 | ChiTotal | total:m1127_39 | | ir.cpp:1127:18:1127:18 | Condition | r1127_30 | | ir.cpp:1127:18:1127:18 | Load | m1126_6 | | ir.cpp:1127:18:1127:18 | Load | m1127_6 | | ir.cpp:1127:18:1127:18 | Load | m1127_6 | | ir.cpp:1127:18:1127:18 | Phi | from 0:m1127_12 | | ir.cpp:1127:18:1127:18 | Phi | from 0:~m1126_4 | -| ir.cpp:1127:18:1127:18 | Phi | from 4:m1127_50 | -| ir.cpp:1127:18:1127:18 | Phi | from 4:~m1127_47 | +| ir.cpp:1127:18:1127:18 | Phi | from 4:m1127_44 | +| ir.cpp:1127:18:1127:18 | Phi | from 4:~m1127_27 | | ir.cpp:1127:18:1127:18 | SideEffect | m1127_19 | | ir.cpp:1127:18:1127:18 | SideEffect | ~m1127_20 | -| ir.cpp:1127:18:1127:18 | SideEffect | ~m1127_27 | -| ir.cpp:1127:18:1127:18 | SideEffect | ~m1127_32 | -| ir.cpp:1127:18:1127:18 | SideEffect | ~m1127_39 | | ir.cpp:1127:18:1127:18 | StoreValue | r1127_5 | | ir.cpp:1127:18:1127:18 | StoreValue | r1127_11 | | ir.cpp:1127:18:1127:18 | StoreValue | r1127_17 | @@ -6356,61 +6347,52 @@ | ir.cpp:1127:18:1127:18 | Unary | r1127_15 | | ir.cpp:1127:18:1127:18 | Unary | r1127_21 | | ir.cpp:1127:18:1127:18 | Unary | r1127_24 | -| ir.cpp:1127:18:1127:18 | Unary | r1127_35 | -| ir.cpp:1127:18:1127:18 | Unary | r1127_45 | -| ir.cpp:1127:18:1127:19 | Load | ~m1127_39 | -| ir.cpp:1127:18:1127:19 | StoreValue | r1127_40 | +| ir.cpp:1127:18:1127:18 | Unary | r1127_33 | +| ir.cpp:1127:18:1127:18 | Unary | r1127_41 | +| ir.cpp:1127:18:1127:19 | Load | ~m1127_27 | +| ir.cpp:1127:18:1127:19 | StoreValue | r1127_36 | | ir.cpp:1128:13:1128:13 | Address | &:r1128_1 | | ir.cpp:1128:13:1128:13 | Left | r1128_2 | -| ir.cpp:1128:13:1128:13 | Load | m1127_41 | +| ir.cpp:1128:13:1128:13 | Load | m1127_37 | | ir.cpp:1128:13:1128:17 | Condition | r1128_4 | | ir.cpp:1128:17:1128:17 | Right | r1128_3 | | ir.cpp:1133:5:1133:5 | Address | &:r1133_1 | | ir.cpp:1133:5:1133:5 | Address | &:r1133_7 | | ir.cpp:1133:5:1133:5 | Address | &:r1133_13 | -| ir.cpp:1133:21:1133:21 | Address | &:r1133_43 | +| ir.cpp:1133:21:1133:21 | Address | &:r1133_39 | | ir.cpp:1133:25:1133:25 | Address | &:r1133_2 | | ir.cpp:1133:25:1133:25 | Address | &:r1133_8 | | ir.cpp:1133:25:1133:25 | Address | &:r1133_14 | -| ir.cpp:1133:25:1133:25 | Address | &:r1133_34 | -| ir.cpp:1133:25:1133:25 | Address | &:r1133_34 | +| ir.cpp:1133:25:1133:25 | Address | &:r1133_32 | +| ir.cpp:1133:25:1133:25 | Address | &:r1133_32 | | ir.cpp:1133:25:1133:25 | Arg(this) | this:r0_15 | | ir.cpp:1133:25:1133:25 | Arg(this) | this:r0_17 | | ir.cpp:1133:25:1133:25 | Arg(this) | this:r0_19 | | ir.cpp:1133:25:1133:25 | Arg(this) | this:r0_20 | | ir.cpp:1133:25:1133:25 | Arg(this) | this:r0_27 | -| ir.cpp:1133:25:1133:25 | Arg(this) | this:r1133_34 | +| ir.cpp:1133:25:1133:25 | Arg(this) | this:r1133_32 | | ir.cpp:1133:25:1133:25 | CallTarget | func:r1133_10 | | ir.cpp:1133:25:1133:25 | CallTarget | func:r1133_16 | | ir.cpp:1133:25:1133:25 | CallTarget | func:r1133_22 | | ir.cpp:1133:25:1133:25 | CallTarget | func:r1133_23 | -| ir.cpp:1133:25:1133:25 | CallTarget | func:r1133_35 | -| ir.cpp:1133:25:1133:25 | CallTarget | func:r1133_45 | +| ir.cpp:1133:25:1133:25 | CallTarget | func:r1133_33 | +| ir.cpp:1133:25:1133:25 | CallTarget | func:r1133_41 | | ir.cpp:1133:25:1133:25 | ChiPartial | partial:m1133_26 | | ir.cpp:1133:25:1133:25 | ChiPartial | partial:m1133_28 | -| ir.cpp:1133:25:1133:25 | ChiPartial | partial:m1133_31 | -| ir.cpp:1133:25:1133:25 | ChiPartial | partial:m1133_37 | -| ir.cpp:1133:25:1133:25 | ChiPartial | partial:m1133_40 | -| ir.cpp:1133:25:1133:25 | ChiPartial | partial:m1133_47 | +| ir.cpp:1133:25:1133:25 | ChiPartial | partial:m1133_36 | | ir.cpp:1133:25:1133:25 | ChiTotal | total:m0_21 | | ir.cpp:1133:25:1133:25 | ChiTotal | total:m1133_19 | | ir.cpp:1133:25:1133:25 | ChiTotal | total:m1133_20 | -| ir.cpp:1133:25:1133:25 | ChiTotal | total:m1133_27 | -| ir.cpp:1133:25:1133:25 | ChiTotal | total:m1133_32 | -| ir.cpp:1133:25:1133:25 | ChiTotal | total:m1133_48 | | ir.cpp:1133:25:1133:25 | Condition | r1133_30 | | ir.cpp:1133:25:1133:25 | Load | m1126_6 | | ir.cpp:1133:25:1133:25 | Load | m1133_6 | | ir.cpp:1133:25:1133:25 | Load | m1133_6 | | ir.cpp:1133:25:1133:25 | Phi | from 5:m1133_12 | -| ir.cpp:1133:25:1133:25 | Phi | from 5:~m1127_32 | -| ir.cpp:1133:25:1133:25 | Phi | from 7:m1133_41 | -| ir.cpp:1133:25:1133:25 | Phi | from 7:~m1133_38 | +| ir.cpp:1133:25:1133:25 | Phi | from 5:~m1127_27 | +| ir.cpp:1133:25:1133:25 | Phi | from 7:m1133_37 | +| ir.cpp:1133:25:1133:25 | Phi | from 7:~m1133_27 | | ir.cpp:1133:25:1133:25 | SideEffect | m1133_19 | | ir.cpp:1133:25:1133:25 | SideEffect | ~m1133_20 | -| ir.cpp:1133:25:1133:25 | SideEffect | ~m1133_27 | -| ir.cpp:1133:25:1133:25 | SideEffect | ~m1133_32 | -| ir.cpp:1133:25:1133:25 | SideEffect | ~m1133_48 | | ir.cpp:1133:25:1133:25 | StoreValue | r1133_5 | | ir.cpp:1133:25:1133:25 | StoreValue | r1133_11 | | ir.cpp:1133:25:1133:25 | StoreValue | r1133_17 | @@ -6420,21 +6402,19 @@ | ir.cpp:1133:25:1133:25 | Unary | r1133_15 | | ir.cpp:1133:25:1133:25 | Unary | r1133_21 | | ir.cpp:1133:25:1133:25 | Unary | r1133_24 | -| ir.cpp:1133:25:1133:25 | Unary | r1133_36 | -| ir.cpp:1133:25:1133:25 | Unary | r1133_44 | -| ir.cpp:1133:25:1133:25 | Unary | r1133_46 | -| ir.cpp:1133:25:1133:26 | StoreValue | r1133_51 | -| ir.cpp:1133:25:1133:26 | Unary | r1133_49 | -| ir.cpp:1133:25:1133:26 | Unary | r1133_50 | +| ir.cpp:1133:25:1133:25 | Unary | r1133_34 | +| ir.cpp:1133:25:1133:25 | Unary | r1133_40 | +| ir.cpp:1133:25:1133:25 | Unary | r1133_42 | +| ir.cpp:1133:25:1133:26 | StoreValue | r1133_45 | +| ir.cpp:1133:25:1133:26 | Unary | r1133_43 | +| ir.cpp:1133:25:1133:26 | Unary | r1133_44 | | ir.cpp:1134:13:1134:13 | Address | &:r1134_1 | | ir.cpp:1134:13:1134:13 | Address | &:r1134_2 | | ir.cpp:1134:13:1134:13 | Left | r1134_3 | -| ir.cpp:1134:13:1134:13 | Load | m1133_52 | -| ir.cpp:1134:13:1134:13 | Load | ~m1133_48 | +| ir.cpp:1134:13:1134:13 | Load | m1133_46 | +| ir.cpp:1134:13:1134:13 | Load | ~m1133_27 | | ir.cpp:1134:13:1134:17 | Condition | r1134_5 | | ir.cpp:1134:17:1134:17 | Right | r1134_4 | -| ir.cpp:1137:5:1137:5 | Phi | from 6:~m1133_32 | -| ir.cpp:1137:5:1137:5 | Phi | from 9:~m1133_48 | | ir.cpp:1157:5:1157:11 | Address | &:r1157_7 | | ir.cpp:1157:5:1157:11 | ChiPartial | partial:m1157_3 | | ir.cpp:1157:5:1157:11 | ChiTotal | total:m1157_2 | @@ -11230,97 +11210,88 @@ | ir.cpp:2215:45:2215:46 | ChiTotal | total:m2214_6 | | ir.cpp:2215:45:2215:46 | ChiTotal | total:m2215_2 | | ir.cpp:2215:45:2215:46 | SideEffect | ~m2214_6 | -| ir.cpp:2215:69:2215:69 | Address | &:r2215_45 | -| ir.cpp:2215:69:2215:69 | Address | &:r2215_53 | -| ir.cpp:2215:69:2215:69 | Address | &:r2215_53 | -| ir.cpp:2215:69:2215:69 | Arg(this) | this:r2215_53 | -| ir.cpp:2215:69:2215:69 | CallTarget | func:r2215_54 | -| ir.cpp:2215:69:2215:69 | ChiPartial | partial:m2215_56 | -| ir.cpp:2215:69:2215:69 | ChiPartial | partial:m2215_59 | +| ir.cpp:2215:69:2215:69 | Address | &:r2215_43 | +| ir.cpp:2215:69:2215:69 | Address | &:r2215_49 | +| ir.cpp:2215:69:2215:69 | Address | &:r2215_49 | +| ir.cpp:2215:69:2215:69 | Arg(this) | this:r2215_49 | +| ir.cpp:2215:69:2215:69 | CallTarget | func:r2215_50 | +| ir.cpp:2215:69:2215:69 | ChiPartial | partial:m2215_52 | +| ir.cpp:2215:69:2215:69 | ChiPartial | partial:m2215_55 | | ir.cpp:2215:69:2215:69 | ChiTotal | total:m2216_6 | | ir.cpp:2215:69:2215:69 | ChiTotal | total:m2216_9 | | ir.cpp:2215:69:2215:69 | SideEffect | m2216_9 | | ir.cpp:2215:69:2215:69 | SideEffect | ~m2216_6 | | ir.cpp:2215:73:2215:73 | Address | &:r2215_19 | | ir.cpp:2215:73:2215:73 | Address | &:r2215_25 | -| ir.cpp:2215:73:2215:73 | Address | &:r2215_48 | -| ir.cpp:2215:73:2215:73 | Address | &:r2215_61 | -| ir.cpp:2215:73:2215:73 | Address | &:r2215_61 | +| ir.cpp:2215:73:2215:73 | Address | &:r2215_46 | +| ir.cpp:2215:73:2215:73 | Address | &:r2215_57 | +| ir.cpp:2215:73:2215:73 | Address | &:r2215_57 | | ir.cpp:2215:73:2215:73 | Arg(this) | this:r0_2 | | ir.cpp:2215:73:2215:73 | Arg(this) | this:r0_5 | | ir.cpp:2215:73:2215:73 | Arg(this) | this:r0_7 | | ir.cpp:2215:73:2215:73 | Arg(this) | this:r0_8 | | ir.cpp:2215:73:2215:73 | Arg(this) | this:r0_15 | -| ir.cpp:2215:73:2215:73 | Arg(this) | this:r2215_61 | +| ir.cpp:2215:73:2215:73 | Arg(this) | this:r2215_57 | | ir.cpp:2215:73:2215:73 | CallTarget | func:r2215_21 | | ir.cpp:2215:73:2215:73 | CallTarget | func:r2215_27 | | ir.cpp:2215:73:2215:73 | CallTarget | func:r2215_33 | | ir.cpp:2215:73:2215:73 | CallTarget | func:r2215_34 | -| ir.cpp:2215:73:2215:73 | CallTarget | func:r2215_47 | -| ir.cpp:2215:73:2215:73 | CallTarget | func:r2215_62 | +| ir.cpp:2215:73:2215:73 | CallTarget | func:r2215_45 | +| ir.cpp:2215:73:2215:73 | CallTarget | func:r2215_58 | | ir.cpp:2215:73:2215:73 | ChiPartial | partial:m2215_37 | | ir.cpp:2215:73:2215:73 | ChiPartial | partial:m2215_39 | -| ir.cpp:2215:73:2215:73 | ChiPartial | partial:m2215_42 | -| ir.cpp:2215:73:2215:73 | ChiPartial | partial:m2215_49 | -| ir.cpp:2215:73:2215:73 | ChiPartial | partial:m2215_64 | -| ir.cpp:2215:73:2215:73 | ChiPartial | partial:m2215_67 | +| ir.cpp:2215:73:2215:73 | ChiPartial | partial:m2215_61 | | ir.cpp:2215:73:2215:73 | ChiTotal | total:m0_9 | | ir.cpp:2215:73:2215:73 | ChiTotal | total:m2215_30 | | ir.cpp:2215:73:2215:73 | ChiTotal | total:m2215_31 | -| ir.cpp:2215:73:2215:73 | ChiTotal | total:m2215_38 | -| ir.cpp:2215:73:2215:73 | ChiTotal | total:m2215_43 | -| ir.cpp:2215:73:2215:73 | ChiTotal | total:m2215_57 | | ir.cpp:2215:73:2215:73 | Condition | r2215_41 | | ir.cpp:2215:73:2215:73 | Load | m2215_17 | | ir.cpp:2215:73:2215:73 | Load | m2215_17 | | ir.cpp:2215:73:2215:73 | Phi | from 7:m2215_23 | | ir.cpp:2215:73:2215:73 | Phi | from 7:~m2215_11 | -| ir.cpp:2215:73:2215:73 | Phi | from 9:m2215_68 | -| ir.cpp:2215:73:2215:73 | Phi | from 9:~m2215_65 | +| ir.cpp:2215:73:2215:73 | Phi | from 9:m2215_62 | +| ir.cpp:2215:73:2215:73 | Phi | from 9:~m2215_53 | | ir.cpp:2215:73:2215:73 | SideEffect | m2215_30 | | ir.cpp:2215:73:2215:73 | SideEffect | ~m2215_31 | -| ir.cpp:2215:73:2215:73 | SideEffect | ~m2215_38 | -| ir.cpp:2215:73:2215:73 | SideEffect | ~m2215_43 | -| ir.cpp:2215:73:2215:73 | SideEffect | ~m2215_57 | | ir.cpp:2215:73:2215:73 | StoreValue | r2215_22 | | ir.cpp:2215:73:2215:73 | StoreValue | r2215_28 | | ir.cpp:2215:73:2215:73 | Unary | r2215_20 | | ir.cpp:2215:73:2215:73 | Unary | r2215_26 | | ir.cpp:2215:73:2215:73 | Unary | r2215_32 | | ir.cpp:2215:73:2215:73 | Unary | r2215_35 | -| ir.cpp:2215:73:2215:73 | Unary | r2215_46 | -| ir.cpp:2215:73:2215:73 | Unary | r2215_63 | +| ir.cpp:2215:73:2215:73 | Unary | r2215_44 | +| ir.cpp:2215:73:2215:73 | Unary | r2215_59 | | ir.cpp:2215:73:2215:74 | StoreValue | r2215_16 | | ir.cpp:2215:73:2215:74 | Unary | r2215_15 | -| ir.cpp:2215:73:2215:75 | Load | ~m2215_50 | -| ir.cpp:2215:73:2215:75 | StoreValue | r2215_51 | +| ir.cpp:2215:73:2215:75 | Load | ~m2215_38 | +| ir.cpp:2215:73:2215:75 | StoreValue | r2215_47 | | ir.cpp:2216:7:2216:7 | Address | &:r2216_1 | | ir.cpp:2216:7:2216:7 | Address | &:r2216_1 | | ir.cpp:2216:7:2216:7 | Arg(this) | this:r2216_1 | | ir.cpp:2216:7:2216:7 | ChiPartial | partial:m2216_8 | -| ir.cpp:2216:7:2216:7 | ChiTotal | total:m2215_52 | -| ir.cpp:2216:7:2216:7 | SideEffect | m2215_52 | +| ir.cpp:2216:7:2216:7 | ChiTotal | total:m2215_48 | +| ir.cpp:2216:7:2216:7 | SideEffect | m2215_48 | | ir.cpp:2216:9:2216:13 | CallTarget | func:r2216_2 | | ir.cpp:2216:9:2216:13 | ChiPartial | partial:m2216_5 | -| ir.cpp:2216:9:2216:13 | ChiTotal | total:m2215_50 | -| ir.cpp:2216:9:2216:13 | SideEffect | ~m2215_50 | +| ir.cpp:2216:9:2216:13 | ChiTotal | total:m2215_38 | +| ir.cpp:2216:9:2216:13 | SideEffect | ~m2215_38 | | ir.cpp:2216:15:2216:17 | Arg(0) | 0:r2216_3 | | ir.cpp:2218:5:2218:5 | Address | &:r2218_14 | | ir.cpp:2218:5:2218:5 | Address | &:r2218_18 | | ir.cpp:2218:5:2218:5 | Address | &:r2218_24 | | ir.cpp:2218:42:2218:43 | Address | &:r2218_1 | | ir.cpp:2218:42:2218:43 | Address | &:r2218_1 | -| ir.cpp:2218:42:2218:43 | Address | &:r2218_61 | -| ir.cpp:2218:42:2218:43 | Address | &:r2218_61 | +| ir.cpp:2218:42:2218:43 | Address | &:r2218_57 | +| ir.cpp:2218:42:2218:43 | Address | &:r2218_57 | | ir.cpp:2218:42:2218:43 | Arg(this) | this:r2218_1 | -| ir.cpp:2218:42:2218:43 | Arg(this) | this:r2218_61 | -| ir.cpp:2218:42:2218:43 | CallTarget | func:r2218_62 | -| ir.cpp:2218:42:2218:43 | ChiPartial | partial:m2218_64 | -| ir.cpp:2218:42:2218:43 | ChiPartial | partial:m2218_67 | +| ir.cpp:2218:42:2218:43 | Arg(this) | this:r2218_57 | +| ir.cpp:2218:42:2218:43 | CallTarget | func:r2218_58 | +| ir.cpp:2218:42:2218:43 | ChiPartial | partial:m2218_60 | +| ir.cpp:2218:42:2218:43 | ChiPartial | partial:m2218_63 | | ir.cpp:2218:42:2218:43 | ChiTotal | total:m2218_13 | -| ir.cpp:2218:42:2218:43 | ChiTotal | total:m2218_57 | +| ir.cpp:2218:42:2218:43 | ChiTotal | total:m2218_53 | | ir.cpp:2218:42:2218:43 | SideEffect | m2218_13 | -| ir.cpp:2218:42:2218:43 | SideEffect | ~m2218_57 | +| ir.cpp:2218:42:2218:43 | SideEffect | ~m2218_53 | | ir.cpp:2218:45:2218:45 | Address | &:r2218_4 | | ir.cpp:2218:45:2218:45 | Address | &:r2218_4 | | ir.cpp:2218:45:2218:45 | Address | &:r2218_5 | @@ -11331,22 +11302,22 @@ | ir.cpp:2218:45:2218:46 | CallTarget | func:r2218_3 | | ir.cpp:2218:45:2218:46 | ChiPartial | partial:m2218_10 | | ir.cpp:2218:45:2218:46 | ChiPartial | partial:m2218_12 | -| ir.cpp:2218:45:2218:46 | ChiTotal | total:m2215_43 | +| ir.cpp:2218:45:2218:46 | ChiTotal | total:m2215_38 | | ir.cpp:2218:45:2218:46 | ChiTotal | total:m2218_2 | -| ir.cpp:2218:45:2218:46 | SideEffect | ~m2215_43 | -| ir.cpp:2218:69:2218:69 | Address | &:r2218_45 | -| ir.cpp:2218:69:2218:69 | Address | &:r2218_53 | -| ir.cpp:2218:69:2218:69 | Address | &:r2218_53 | -| ir.cpp:2218:69:2218:69 | Address | &:r2218_69 | -| ir.cpp:2218:69:2218:69 | Address | &:r2218_69 | -| ir.cpp:2218:69:2218:69 | Arg(this) | this:r2218_53 | -| ir.cpp:2218:69:2218:69 | Arg(this) | this:r2218_69 | -| ir.cpp:2218:69:2218:69 | CallTarget | func:r2218_54 | -| ir.cpp:2218:69:2218:69 | CallTarget | func:r2218_70 | -| ir.cpp:2218:69:2218:69 | ChiPartial | partial:m2218_56 | -| ir.cpp:2218:69:2218:69 | ChiPartial | partial:m2218_59 | -| ir.cpp:2218:69:2218:69 | ChiPartial | partial:m2218_72 | -| ir.cpp:2218:69:2218:69 | ChiPartial | partial:m2218_75 | +| ir.cpp:2218:45:2218:46 | SideEffect | ~m2215_38 | +| ir.cpp:2218:69:2218:69 | Address | &:r2218_43 | +| ir.cpp:2218:69:2218:69 | Address | &:r2218_49 | +| ir.cpp:2218:69:2218:69 | Address | &:r2218_49 | +| ir.cpp:2218:69:2218:69 | Address | &:r2218_65 | +| ir.cpp:2218:69:2218:69 | Address | &:r2218_65 | +| ir.cpp:2218:69:2218:69 | Arg(this) | this:r2218_49 | +| ir.cpp:2218:69:2218:69 | Arg(this) | this:r2218_65 | +| ir.cpp:2218:69:2218:69 | CallTarget | func:r2218_50 | +| ir.cpp:2218:69:2218:69 | CallTarget | func:r2218_66 | +| ir.cpp:2218:69:2218:69 | ChiPartial | partial:m2218_52 | +| ir.cpp:2218:69:2218:69 | ChiPartial | partial:m2218_55 | +| ir.cpp:2218:69:2218:69 | ChiPartial | partial:m2218_68 | +| ir.cpp:2218:69:2218:69 | ChiPartial | partial:m2218_71 | | ir.cpp:2218:69:2218:69 | ChiTotal | total:m2220_5 | | ir.cpp:2218:69:2218:69 | ChiTotal | total:m2220_5 | | ir.cpp:2218:69:2218:69 | ChiTotal | total:m2220_8 | @@ -11357,67 +11328,58 @@ | ir.cpp:2218:69:2218:69 | SideEffect | ~m2220_5 | | ir.cpp:2218:73:2218:73 | Address | &:r2218_19 | | ir.cpp:2218:73:2218:73 | Address | &:r2218_25 | -| ir.cpp:2218:73:2218:73 | Address | &:r2218_48 | -| ir.cpp:2218:73:2218:73 | Address | &:r2218_77 | -| ir.cpp:2218:73:2218:73 | Address | &:r2218_77 | +| ir.cpp:2218:73:2218:73 | Address | &:r2218_46 | +| ir.cpp:2218:73:2218:73 | Address | &:r2218_73 | +| ir.cpp:2218:73:2218:73 | Address | &:r2218_73 | | ir.cpp:2218:73:2218:73 | Arg(this) | this:r0_18 | | ir.cpp:2218:73:2218:73 | Arg(this) | this:r0_21 | | ir.cpp:2218:73:2218:73 | Arg(this) | this:r0_23 | | ir.cpp:2218:73:2218:73 | Arg(this) | this:r0_24 | | ir.cpp:2218:73:2218:73 | Arg(this) | this:r0_31 | -| ir.cpp:2218:73:2218:73 | Arg(this) | this:r2218_77 | +| ir.cpp:2218:73:2218:73 | Arg(this) | this:r2218_73 | | ir.cpp:2218:73:2218:73 | CallTarget | func:r2218_21 | | ir.cpp:2218:73:2218:73 | CallTarget | func:r2218_27 | | ir.cpp:2218:73:2218:73 | CallTarget | func:r2218_33 | | ir.cpp:2218:73:2218:73 | CallTarget | func:r2218_34 | -| ir.cpp:2218:73:2218:73 | CallTarget | func:r2218_47 | -| ir.cpp:2218:73:2218:73 | CallTarget | func:r2218_78 | +| ir.cpp:2218:73:2218:73 | CallTarget | func:r2218_45 | +| ir.cpp:2218:73:2218:73 | CallTarget | func:r2218_74 | | ir.cpp:2218:73:2218:73 | ChiPartial | partial:m2218_37 | | ir.cpp:2218:73:2218:73 | ChiPartial | partial:m2218_39 | -| ir.cpp:2218:73:2218:73 | ChiPartial | partial:m2218_42 | -| ir.cpp:2218:73:2218:73 | ChiPartial | partial:m2218_49 | -| ir.cpp:2218:73:2218:73 | ChiPartial | partial:m2218_80 | -| ir.cpp:2218:73:2218:73 | ChiPartial | partial:m2218_83 | +| ir.cpp:2218:73:2218:73 | ChiPartial | partial:m2218_77 | | ir.cpp:2218:73:2218:73 | ChiTotal | total:m0_25 | | ir.cpp:2218:73:2218:73 | ChiTotal | total:m2218_30 | | ir.cpp:2218:73:2218:73 | ChiTotal | total:m2218_31 | -| ir.cpp:2218:73:2218:73 | ChiTotal | total:m2218_38 | -| ir.cpp:2218:73:2218:73 | ChiTotal | total:m2218_43 | -| ir.cpp:2218:73:2218:73 | ChiTotal | total:m2218_73 | | ir.cpp:2218:73:2218:73 | Condition | r2218_41 | | ir.cpp:2218:73:2218:73 | Load | m2218_17 | | ir.cpp:2218:73:2218:73 | Load | m2218_17 | | ir.cpp:2218:73:2218:73 | Phi | from 10:m2218_23 | | ir.cpp:2218:73:2218:73 | Phi | from 10:~m2218_11 | -| ir.cpp:2218:73:2218:73 | Phi | from 14:m2218_84 | -| ir.cpp:2218:73:2218:73 | Phi | from 14:~m2218_81 | +| ir.cpp:2218:73:2218:73 | Phi | from 14:m2218_78 | +| ir.cpp:2218:73:2218:73 | Phi | from 14:~m2218_69 | | ir.cpp:2218:73:2218:73 | SideEffect | m2218_30 | | ir.cpp:2218:73:2218:73 | SideEffect | ~m2218_31 | -| ir.cpp:2218:73:2218:73 | SideEffect | ~m2218_38 | -| ir.cpp:2218:73:2218:73 | SideEffect | ~m2218_43 | -| ir.cpp:2218:73:2218:73 | SideEffect | ~m2218_73 | | ir.cpp:2218:73:2218:73 | StoreValue | r2218_22 | | ir.cpp:2218:73:2218:73 | StoreValue | r2218_28 | | ir.cpp:2218:73:2218:73 | Unary | r2218_20 | | ir.cpp:2218:73:2218:73 | Unary | r2218_26 | | ir.cpp:2218:73:2218:73 | Unary | r2218_32 | | ir.cpp:2218:73:2218:73 | Unary | r2218_35 | -| ir.cpp:2218:73:2218:73 | Unary | r2218_46 | -| ir.cpp:2218:73:2218:73 | Unary | r2218_79 | +| ir.cpp:2218:73:2218:73 | Unary | r2218_44 | +| ir.cpp:2218:73:2218:73 | Unary | r2218_75 | | ir.cpp:2218:73:2218:74 | StoreValue | r2218_16 | | ir.cpp:2218:73:2218:74 | Unary | r2218_15 | -| ir.cpp:2218:73:2218:75 | Load | ~m2218_50 | -| ir.cpp:2218:73:2218:75 | StoreValue | r2218_51 | +| ir.cpp:2218:73:2218:75 | Load | ~m2218_38 | +| ir.cpp:2218:73:2218:75 | StoreValue | r2218_47 | | ir.cpp:2219:7:2219:7 | Address | &:r2219_1 | | ir.cpp:2219:7:2219:7 | Address | &:r2219_1 | | ir.cpp:2219:7:2219:7 | Arg(this) | this:r2219_1 | | ir.cpp:2219:7:2219:7 | ChiPartial | partial:m2219_8 | -| ir.cpp:2219:7:2219:7 | ChiTotal | total:m2218_52 | -| ir.cpp:2219:7:2219:7 | SideEffect | m2218_52 | +| ir.cpp:2219:7:2219:7 | ChiTotal | total:m2218_48 | +| ir.cpp:2219:7:2219:7 | SideEffect | m2218_48 | | ir.cpp:2219:9:2219:13 | CallTarget | func:r2219_2 | | ir.cpp:2219:9:2219:13 | ChiPartial | partial:m2219_5 | -| ir.cpp:2219:9:2219:13 | ChiTotal | total:m2218_50 | -| ir.cpp:2219:9:2219:13 | SideEffect | ~m2218_50 | +| ir.cpp:2219:9:2219:13 | ChiTotal | total:m2218_38 | +| ir.cpp:2219:9:2219:13 | SideEffect | ~m2218_38 | | ir.cpp:2219:15:2219:17 | Arg(0) | 0:r2219_3 | | ir.cpp:2220:11:2220:11 | Address | &:r2220_1 | | ir.cpp:2220:11:2220:11 | Address | &:r2220_1 | @@ -11438,81 +11400,72 @@ | ir.cpp:2224:5:2224:5 | Address | &:r2224_20 | | ir.cpp:2224:26:2224:27 | Address | &:r2224_1 | | ir.cpp:2224:26:2224:27 | Address | &:r2224_1 | -| ir.cpp:2224:26:2224:27 | Address | &:r2224_58 | -| ir.cpp:2224:26:2224:27 | Address | &:r2224_58 | +| ir.cpp:2224:26:2224:27 | Address | &:r2224_52 | +| ir.cpp:2224:26:2224:27 | Address | &:r2224_52 | | ir.cpp:2224:26:2224:27 | Arg(this) | this:r2224_1 | -| ir.cpp:2224:26:2224:27 | Arg(this) | this:r2224_58 | -| ir.cpp:2224:26:2224:27 | CallTarget | func:r2224_59 | -| ir.cpp:2224:26:2224:27 | ChiPartial | partial:m2224_61 | -| ir.cpp:2224:26:2224:27 | ChiPartial | partial:m2224_64 | +| ir.cpp:2224:26:2224:27 | Arg(this) | this:r2224_52 | +| ir.cpp:2224:26:2224:27 | CallTarget | func:r2224_53 | +| ir.cpp:2224:26:2224:27 | ChiPartial | partial:m2224_55 | +| ir.cpp:2224:26:2224:27 | ChiPartial | partial:m2224_58 | | ir.cpp:2224:26:2224:27 | ChiTotal | total:m2224_9 | -| ir.cpp:2224:26:2224:27 | ChiTotal | total:m2224_55 | +| ir.cpp:2224:26:2224:27 | ChiTotal | total:m2224_34 | | ir.cpp:2224:26:2224:27 | SideEffect | m2224_9 | -| ir.cpp:2224:26:2224:27 | SideEffect | ~m2224_55 | +| ir.cpp:2224:26:2224:27 | SideEffect | ~m2224_34 | | ir.cpp:2224:29:2224:29 | Arg(0) | 0:r2224_4 | | ir.cpp:2224:29:2224:30 | CallTarget | func:r2224_3 | | ir.cpp:2224:29:2224:30 | ChiPartial | partial:m2224_6 | | ir.cpp:2224:29:2224:30 | ChiPartial | partial:m2224_8 | -| ir.cpp:2224:29:2224:30 | ChiTotal | total:m2218_43 | +| ir.cpp:2224:29:2224:30 | ChiTotal | total:m2218_38 | | ir.cpp:2224:29:2224:30 | ChiTotal | total:m2224_2 | -| ir.cpp:2224:29:2224:30 | SideEffect | ~m2218_43 | -| ir.cpp:2224:37:2224:37 | Address | &:r2224_50 | +| ir.cpp:2224:29:2224:30 | SideEffect | ~m2218_38 | +| ir.cpp:2224:37:2224:37 | Address | &:r2224_46 | | ir.cpp:2224:41:2224:41 | Address | &:r2224_15 | | ir.cpp:2224:41:2224:41 | Address | &:r2224_21 | -| ir.cpp:2224:41:2224:41 | Address | &:r2224_41 | -| ir.cpp:2224:41:2224:41 | Address | &:r2224_41 | -| ir.cpp:2224:41:2224:41 | Address | &:r2224_53 | +| ir.cpp:2224:41:2224:41 | Address | &:r2224_39 | +| ir.cpp:2224:41:2224:41 | Address | &:r2224_39 | +| ir.cpp:2224:41:2224:41 | Address | &:r2224_49 | | ir.cpp:2224:41:2224:41 | Arg(this) | this:r0_34 | | ir.cpp:2224:41:2224:41 | Arg(this) | this:r0_37 | | ir.cpp:2224:41:2224:41 | Arg(this) | this:r0_39 | | ir.cpp:2224:41:2224:41 | Arg(this) | this:r0_40 | | ir.cpp:2224:41:2224:41 | Arg(this) | this:r0_47 | -| ir.cpp:2224:41:2224:41 | Arg(this) | this:r2224_41 | +| ir.cpp:2224:41:2224:41 | Arg(this) | this:r2224_39 | | ir.cpp:2224:41:2224:41 | CallTarget | func:r2224_17 | | ir.cpp:2224:41:2224:41 | CallTarget | func:r2224_23 | | ir.cpp:2224:41:2224:41 | CallTarget | func:r2224_29 | | ir.cpp:2224:41:2224:41 | CallTarget | func:r2224_30 | -| ir.cpp:2224:41:2224:41 | CallTarget | func:r2224_42 | -| ir.cpp:2224:41:2224:41 | CallTarget | func:r2224_52 | +| ir.cpp:2224:41:2224:41 | CallTarget | func:r2224_40 | +| ir.cpp:2224:41:2224:41 | CallTarget | func:r2224_48 | | ir.cpp:2224:41:2224:41 | ChiPartial | partial:m2224_33 | | ir.cpp:2224:41:2224:41 | ChiPartial | partial:m2224_35 | -| ir.cpp:2224:41:2224:41 | ChiPartial | partial:m2224_38 | -| ir.cpp:2224:41:2224:41 | ChiPartial | partial:m2224_44 | -| ir.cpp:2224:41:2224:41 | ChiPartial | partial:m2224_47 | -| ir.cpp:2224:41:2224:41 | ChiPartial | partial:m2224_54 | +| ir.cpp:2224:41:2224:41 | ChiPartial | partial:m2224_43 | | ir.cpp:2224:41:2224:41 | ChiTotal | total:m0_41 | | ir.cpp:2224:41:2224:41 | ChiTotal | total:m2224_26 | | ir.cpp:2224:41:2224:41 | ChiTotal | total:m2224_27 | -| ir.cpp:2224:41:2224:41 | ChiTotal | total:m2224_34 | -| ir.cpp:2224:41:2224:41 | ChiTotal | total:m2224_39 | -| ir.cpp:2224:41:2224:41 | ChiTotal | total:m2224_55 | | ir.cpp:2224:41:2224:41 | Condition | r2224_37 | | ir.cpp:2224:41:2224:41 | Load | m2224_13 | | ir.cpp:2224:41:2224:41 | Load | m2224_13 | | ir.cpp:2224:41:2224:41 | Phi | from 15:m2224_19 | | ir.cpp:2224:41:2224:41 | Phi | from 15:~m2224_7 | -| ir.cpp:2224:41:2224:41 | Phi | from 17:m2224_48 | -| ir.cpp:2224:41:2224:41 | Phi | from 17:~m2224_45 | +| ir.cpp:2224:41:2224:41 | Phi | from 17:m2224_44 | +| ir.cpp:2224:41:2224:41 | Phi | from 17:~m2224_34 | | ir.cpp:2224:41:2224:41 | SideEffect | m2224_26 | | ir.cpp:2224:41:2224:41 | SideEffect | ~m2224_27 | -| ir.cpp:2224:41:2224:41 | SideEffect | ~m2224_34 | -| ir.cpp:2224:41:2224:41 | SideEffect | ~m2224_39 | -| ir.cpp:2224:41:2224:41 | SideEffect | ~m2224_55 | | ir.cpp:2224:41:2224:41 | StoreValue | r2224_18 | | ir.cpp:2224:41:2224:41 | StoreValue | r2224_24 | | ir.cpp:2224:41:2224:41 | Unary | r2224_16 | | ir.cpp:2224:41:2224:41 | Unary | r2224_22 | | ir.cpp:2224:41:2224:41 | Unary | r2224_28 | | ir.cpp:2224:41:2224:41 | Unary | r2224_31 | -| ir.cpp:2224:41:2224:41 | Unary | r2224_43 | -| ir.cpp:2224:41:2224:41 | Unary | r2224_51 | +| ir.cpp:2224:41:2224:41 | Unary | r2224_41 | +| ir.cpp:2224:41:2224:41 | Unary | r2224_47 | | ir.cpp:2224:41:2224:42 | StoreValue | r2224_12 | | ir.cpp:2224:41:2224:42 | Unary | r2224_11 | -| ir.cpp:2224:41:2224:43 | Load | ~m2224_55 | -| ir.cpp:2224:41:2224:43 | StoreValue | r2224_56 | +| ir.cpp:2224:41:2224:43 | Load | ~m2224_34 | +| ir.cpp:2224:41:2224:43 | StoreValue | r2224_50 | | ir.cpp:2225:11:2225:11 | Address | &:r2225_1 | | ir.cpp:2225:11:2225:11 | Left | r2225_2 | -| ir.cpp:2225:11:2225:11 | Load | m2224_57 | +| ir.cpp:2225:11:2225:11 | Load | m2224_51 | | ir.cpp:2225:11:2225:16 | Condition | r2225_4 | | ir.cpp:2225:16:2225:16 | Right | r2225_3 | | ir.cpp:2229:5:2229:5 | Address | &:r2229_14 | @@ -11531,82 +11484,73 @@ | ir.cpp:2229:45:2229:46 | CallTarget | func:r2229_3 | | ir.cpp:2229:45:2229:46 | ChiPartial | partial:m2229_10 | | ir.cpp:2229:45:2229:46 | ChiPartial | partial:m2229_12 | -| ir.cpp:2229:45:2229:46 | ChiTotal | total:m2224_39 | +| ir.cpp:2229:45:2229:46 | ChiTotal | total:m2224_34 | | ir.cpp:2229:45:2229:46 | ChiTotal | total:m2229_2 | -| ir.cpp:2229:45:2229:46 | SideEffect | ~m2224_39 | -| ir.cpp:2229:69:2229:69 | Address | &:r2229_45 | -| ir.cpp:2229:69:2229:69 | Address | &:r2229_53 | -| ir.cpp:2229:69:2229:69 | Address | &:r2229_53 | -| ir.cpp:2229:69:2229:69 | Arg(this) | this:r2229_53 | -| ir.cpp:2229:69:2229:69 | CallTarget | func:r2229_54 | -| ir.cpp:2229:69:2229:69 | ChiPartial | partial:m2229_56 | -| ir.cpp:2229:69:2229:69 | ChiPartial | partial:m2229_59 | -| ir.cpp:2229:69:2229:69 | ChiTotal | total:m2229_52 | +| ir.cpp:2229:45:2229:46 | SideEffect | ~m2224_34 | +| ir.cpp:2229:69:2229:69 | Address | &:r2229_43 | +| ir.cpp:2229:69:2229:69 | Address | &:r2229_49 | +| ir.cpp:2229:69:2229:69 | Address | &:r2229_49 | +| ir.cpp:2229:69:2229:69 | Arg(this) | this:r2229_49 | +| ir.cpp:2229:69:2229:69 | CallTarget | func:r2229_50 | +| ir.cpp:2229:69:2229:69 | ChiPartial | partial:m2229_52 | +| ir.cpp:2229:69:2229:69 | ChiPartial | partial:m2229_55 | +| ir.cpp:2229:69:2229:69 | ChiTotal | total:m2229_48 | | ir.cpp:2229:69:2229:69 | ChiTotal | total:m2232_13 | -| ir.cpp:2229:69:2229:69 | SideEffect | m2229_52 | +| ir.cpp:2229:69:2229:69 | SideEffect | m2229_48 | | ir.cpp:2229:69:2229:69 | SideEffect | ~m2232_13 | | ir.cpp:2229:73:2229:73 | Address | &:r2229_19 | | ir.cpp:2229:73:2229:73 | Address | &:r2229_25 | -| ir.cpp:2229:73:2229:73 | Address | &:r2229_48 | -| ir.cpp:2229:73:2229:73 | Address | &:r2229_61 | -| ir.cpp:2229:73:2229:73 | Address | &:r2229_61 | +| ir.cpp:2229:73:2229:73 | Address | &:r2229_46 | +| ir.cpp:2229:73:2229:73 | Address | &:r2229_57 | +| ir.cpp:2229:73:2229:73 | Address | &:r2229_57 | | ir.cpp:2229:73:2229:73 | Arg(this) | this:r0_50 | | ir.cpp:2229:73:2229:73 | Arg(this) | this:r0_53 | | ir.cpp:2229:73:2229:73 | Arg(this) | this:r0_55 | | ir.cpp:2229:73:2229:73 | Arg(this) | this:r0_56 | | ir.cpp:2229:73:2229:73 | Arg(this) | this:r0_63 | -| ir.cpp:2229:73:2229:73 | Arg(this) | this:r2229_61 | +| ir.cpp:2229:73:2229:73 | Arg(this) | this:r2229_57 | | ir.cpp:2229:73:2229:73 | CallTarget | func:r2229_21 | | ir.cpp:2229:73:2229:73 | CallTarget | func:r2229_27 | | ir.cpp:2229:73:2229:73 | CallTarget | func:r2229_33 | | ir.cpp:2229:73:2229:73 | CallTarget | func:r2229_34 | -| ir.cpp:2229:73:2229:73 | CallTarget | func:r2229_47 | -| ir.cpp:2229:73:2229:73 | CallTarget | func:r2229_62 | +| ir.cpp:2229:73:2229:73 | CallTarget | func:r2229_45 | +| ir.cpp:2229:73:2229:73 | CallTarget | func:r2229_58 | | ir.cpp:2229:73:2229:73 | ChiPartial | partial:m2229_37 | | ir.cpp:2229:73:2229:73 | ChiPartial | partial:m2229_39 | -| ir.cpp:2229:73:2229:73 | ChiPartial | partial:m2229_42 | -| ir.cpp:2229:73:2229:73 | ChiPartial | partial:m2229_49 | -| ir.cpp:2229:73:2229:73 | ChiPartial | partial:m2229_64 | -| ir.cpp:2229:73:2229:73 | ChiPartial | partial:m2229_67 | +| ir.cpp:2229:73:2229:73 | ChiPartial | partial:m2229_61 | | ir.cpp:2229:73:2229:73 | ChiTotal | total:m0_57 | | ir.cpp:2229:73:2229:73 | ChiTotal | total:m2229_30 | | ir.cpp:2229:73:2229:73 | ChiTotal | total:m2229_31 | -| ir.cpp:2229:73:2229:73 | ChiTotal | total:m2229_38 | -| ir.cpp:2229:73:2229:73 | ChiTotal | total:m2229_43 | -| ir.cpp:2229:73:2229:73 | ChiTotal | total:m2229_57 | | ir.cpp:2229:73:2229:73 | Condition | r2229_41 | | ir.cpp:2229:73:2229:73 | Load | m2229_17 | | ir.cpp:2229:73:2229:73 | Load | m2229_17 | | ir.cpp:2229:73:2229:73 | Phi | from 20:m2229_23 | | ir.cpp:2229:73:2229:73 | Phi | from 20:~m2229_11 | -| ir.cpp:2229:73:2229:73 | Phi | from 22:m2229_68 | -| ir.cpp:2229:73:2229:73 | Phi | from 22:~m2229_65 | +| ir.cpp:2229:73:2229:73 | Phi | from 22:m2229_62 | +| ir.cpp:2229:73:2229:73 | Phi | from 22:~m2229_53 | | ir.cpp:2229:73:2229:73 | SideEffect | m2229_30 | | ir.cpp:2229:73:2229:73 | SideEffect | ~m2229_31 | -| ir.cpp:2229:73:2229:73 | SideEffect | ~m2229_38 | -| ir.cpp:2229:73:2229:73 | SideEffect | ~m2229_43 | -| ir.cpp:2229:73:2229:73 | SideEffect | ~m2229_57 | | ir.cpp:2229:73:2229:73 | StoreValue | r2229_22 | | ir.cpp:2229:73:2229:73 | StoreValue | r2229_28 | | ir.cpp:2229:73:2229:73 | Unary | r2229_20 | | ir.cpp:2229:73:2229:73 | Unary | r2229_26 | | ir.cpp:2229:73:2229:73 | Unary | r2229_32 | | ir.cpp:2229:73:2229:73 | Unary | r2229_35 | -| ir.cpp:2229:73:2229:73 | Unary | r2229_46 | -| ir.cpp:2229:73:2229:73 | Unary | r2229_63 | +| ir.cpp:2229:73:2229:73 | Unary | r2229_44 | +| ir.cpp:2229:73:2229:73 | Unary | r2229_59 | | ir.cpp:2229:73:2229:74 | StoreValue | r2229_16 | | ir.cpp:2229:73:2229:74 | Unary | r2229_15 | -| ir.cpp:2229:73:2229:75 | Load | ~m2229_50 | -| ir.cpp:2229:73:2229:75 | StoreValue | r2229_51 | +| ir.cpp:2229:73:2229:75 | Load | ~m2229_38 | +| ir.cpp:2229:73:2229:75 | StoreValue | r2229_47 | | ir.cpp:2230:27:2230:28 | Address | &:r2230_1 | | ir.cpp:2230:27:2230:28 | Address | &:r2230_1 | | ir.cpp:2230:27:2230:28 | Arg(this) | this:r2230_1 | | ir.cpp:2230:27:2230:28 | CallTarget | func:r2230_3 | | ir.cpp:2230:27:2230:28 | ChiPartial | partial:m2230_5 | | ir.cpp:2230:27:2230:28 | ChiPartial | partial:m2230_7 | -| ir.cpp:2230:27:2230:28 | ChiTotal | total:m2229_50 | +| ir.cpp:2230:27:2230:28 | ChiTotal | total:m2229_38 | | ir.cpp:2230:27:2230:28 | ChiTotal | total:m2230_2 | -| ir.cpp:2230:27:2230:28 | SideEffect | ~m2229_50 | +| ir.cpp:2230:27:2230:28 | SideEffect | ~m2229_38 | | ir.cpp:2231:27:2231:28 | Address | &:r2231_1 | | ir.cpp:2231:27:2231:28 | Address | &:r2231_1 | | ir.cpp:2231:27:2231:28 | Arg(this) | this:r2231_1 | @@ -11657,15 +11601,15 @@ | ir.cpp:2233:1:2233:1 | ChiTotal | total:m2214_8 | | ir.cpp:2233:1:2233:1 | ChiTotal | total:m2214_8 | | ir.cpp:2233:1:2233:1 | ChiTotal | total:m2214_8 | -| ir.cpp:2233:1:2233:1 | ChiTotal | total:m2218_65 | -| ir.cpp:2233:1:2233:1 | ChiTotal | total:m2224_62 | -| ir.cpp:2233:1:2233:1 | ChiTotal | total:m2229_43 | +| ir.cpp:2233:1:2233:1 | ChiTotal | total:m2218_61 | +| ir.cpp:2233:1:2233:1 | ChiTotal | total:m2224_56 | +| ir.cpp:2233:1:2233:1 | ChiTotal | total:m2229_38 | | ir.cpp:2233:1:2233:1 | SideEffect | m2214_8 | | ir.cpp:2233:1:2233:1 | SideEffect | m2214_8 | | ir.cpp:2233:1:2233:1 | SideEffect | m2214_8 | -| ir.cpp:2233:1:2233:1 | SideEffect | ~m2218_65 | -| ir.cpp:2233:1:2233:1 | SideEffect | ~m2224_62 | -| ir.cpp:2233:1:2233:1 | SideEffect | ~m2229_43 | +| ir.cpp:2233:1:2233:1 | SideEffect | ~m2218_61 | +| ir.cpp:2233:1:2233:1 | SideEffect | ~m2224_56 | +| ir.cpp:2233:1:2233:1 | SideEffect | ~m2229_38 | | ir.cpp:2235:6:2235:38 | ChiPartial | partial:m2235_3 | | ir.cpp:2235:6:2235:38 | ChiTotal | total:m2235_2 | | ir.cpp:2235:6:2235:38 | SideEffect | ~m2238_7 | @@ -12171,74 +12115,65 @@ | ir.cpp:2307:5:2307:5 | Address | &:r2307_1 | | ir.cpp:2307:5:2307:5 | Address | &:r2307_24 | | ir.cpp:2307:5:2307:5 | Address | &:r2307_30 | -| ir.cpp:2307:16:2307:16 | Address | &:r2307_51 | -| ir.cpp:2307:16:2307:16 | Address | &:r2307_51 | -| ir.cpp:2307:16:2307:16 | Address | &:r2307_68 | -| ir.cpp:2307:16:2307:16 | Address | &:r2307_68 | -| ir.cpp:2307:16:2307:16 | Arg(this) | this:r2307_51 | -| ir.cpp:2307:16:2307:16 | Arg(this) | this:r2307_68 | -| ir.cpp:2307:16:2307:16 | CallTarget | func:r2307_53 | -| ir.cpp:2307:16:2307:16 | CallTarget | func:r2307_69 | -| ir.cpp:2307:16:2307:16 | ChiPartial | partial:m2307_63 | -| ir.cpp:2307:16:2307:16 | ChiPartial | partial:m2307_66 | -| ir.cpp:2307:16:2307:16 | ChiPartial | partial:m2307_71 | -| ir.cpp:2307:16:2307:16 | ChiPartial | partial:m2307_74 | -| ir.cpp:2307:16:2307:16 | ChiTotal | total:m2307_52 | -| ir.cpp:2307:16:2307:16 | ChiTotal | total:m2307_58 | -| ir.cpp:2307:16:2307:16 | ChiTotal | total:m2307_67 | +| ir.cpp:2307:16:2307:16 | Address | &:r2307_49 | +| ir.cpp:2307:16:2307:16 | Address | &:r2307_49 | +| ir.cpp:2307:16:2307:16 | Address | &:r2307_64 | +| ir.cpp:2307:16:2307:16 | Address | &:r2307_64 | +| ir.cpp:2307:16:2307:16 | Arg(this) | this:r2307_49 | +| ir.cpp:2307:16:2307:16 | Arg(this) | this:r2307_64 | +| ir.cpp:2307:16:2307:16 | CallTarget | func:r2307_51 | +| ir.cpp:2307:16:2307:16 | CallTarget | func:r2307_65 | +| ir.cpp:2307:16:2307:16 | ChiPartial | partial:m2307_59 | +| ir.cpp:2307:16:2307:16 | ChiPartial | partial:m2307_62 | +| ir.cpp:2307:16:2307:16 | ChiPartial | partial:m2307_67 | +| ir.cpp:2307:16:2307:16 | ChiPartial | partial:m2307_70 | +| ir.cpp:2307:16:2307:16 | ChiTotal | total:m2307_44 | +| ir.cpp:2307:16:2307:16 | ChiTotal | total:m2307_50 | +| ir.cpp:2307:16:2307:16 | ChiTotal | total:m2307_63 | | ir.cpp:2307:16:2307:16 | ChiTotal | total:m2309_5 | -| ir.cpp:2307:16:2307:16 | SideEffect | m2307_67 | -| ir.cpp:2307:16:2307:16 | SideEffect | ~m2307_58 | +| ir.cpp:2307:16:2307:16 | SideEffect | m2307_63 | +| ir.cpp:2307:16:2307:16 | SideEffect | ~m2307_44 | | ir.cpp:2307:16:2307:16 | SideEffect | ~m2309_5 | | ir.cpp:2307:20:2307:20 | Address | &:r2307_25 | | ir.cpp:2307:20:2307:20 | Address | &:r2307_31 | -| ir.cpp:2307:20:2307:20 | Address | &:r2307_76 | -| ir.cpp:2307:20:2307:20 | Address | &:r2307_76 | +| ir.cpp:2307:20:2307:20 | Address | &:r2307_72 | +| ir.cpp:2307:20:2307:20 | Address | &:r2307_72 | | ir.cpp:2307:20:2307:20 | Arg(this) | this:r0_2 | | ir.cpp:2307:20:2307:20 | Arg(this) | this:r0_5 | | ir.cpp:2307:20:2307:20 | Arg(this) | this:r0_7 | | ir.cpp:2307:20:2307:20 | Arg(this) | this:r0_8 | | ir.cpp:2307:20:2307:20 | Arg(this) | this:r0_15 | -| ir.cpp:2307:20:2307:20 | Arg(this) | this:r2307_76 | +| ir.cpp:2307:20:2307:20 | Arg(this) | this:r2307_72 | | ir.cpp:2307:20:2307:20 | CallTarget | func:r2307_27 | | ir.cpp:2307:20:2307:20 | CallTarget | func:r2307_33 | | ir.cpp:2307:20:2307:20 | CallTarget | func:r2307_39 | | ir.cpp:2307:20:2307:20 | CallTarget | func:r2307_40 | -| ir.cpp:2307:20:2307:20 | CallTarget | func:r2307_55 | -| ir.cpp:2307:20:2307:20 | CallTarget | func:r2307_77 | +| ir.cpp:2307:20:2307:20 | CallTarget | func:r2307_53 | +| ir.cpp:2307:20:2307:20 | CallTarget | func:r2307_73 | | ir.cpp:2307:20:2307:20 | ChiPartial | partial:m2307_43 | | ir.cpp:2307:20:2307:20 | ChiPartial | partial:m2307_45 | -| ir.cpp:2307:20:2307:20 | ChiPartial | partial:m2307_48 | -| ir.cpp:2307:20:2307:20 | ChiPartial | partial:m2307_57 | -| ir.cpp:2307:20:2307:20 | ChiPartial | partial:m2307_79 | -| ir.cpp:2307:20:2307:20 | ChiPartial | partial:m2307_82 | +| ir.cpp:2307:20:2307:20 | ChiPartial | partial:m2307_76 | | ir.cpp:2307:20:2307:20 | ChiTotal | total:m0_9 | | ir.cpp:2307:20:2307:20 | ChiTotal | total:m2307_36 | | ir.cpp:2307:20:2307:20 | ChiTotal | total:m2307_37 | -| ir.cpp:2307:20:2307:20 | ChiTotal | total:m2307_44 | -| ir.cpp:2307:20:2307:20 | ChiTotal | total:m2307_49 | -| ir.cpp:2307:20:2307:20 | ChiTotal | total:m2307_72 | | ir.cpp:2307:20:2307:20 | Condition | r2307_47 | | ir.cpp:2307:20:2307:20 | Load | m2307_23 | | ir.cpp:2307:20:2307:20 | Load | m2307_23 | | ir.cpp:2307:20:2307:20 | Phi | from 3:m2307_29 | | ir.cpp:2307:20:2307:20 | Phi | from 3:~m2307_19 | -| ir.cpp:2307:20:2307:20 | Phi | from 5:m2307_83 | -| ir.cpp:2307:20:2307:20 | Phi | from 5:~m2307_80 | +| ir.cpp:2307:20:2307:20 | Phi | from 5:m2307_77 | +| ir.cpp:2307:20:2307:20 | Phi | from 5:~m2307_68 | | ir.cpp:2307:20:2307:20 | SideEffect | m2307_36 | | ir.cpp:2307:20:2307:20 | SideEffect | ~m2307_37 | -| ir.cpp:2307:20:2307:20 | SideEffect | ~m2307_44 | -| ir.cpp:2307:20:2307:20 | SideEffect | ~m2307_49 | -| ir.cpp:2307:20:2307:20 | SideEffect | ~m2307_72 | | ir.cpp:2307:20:2307:20 | StoreValue | r2307_28 | | ir.cpp:2307:20:2307:20 | StoreValue | r2307_34 | | ir.cpp:2307:20:2307:20 | Unary | r2307_26 | | ir.cpp:2307:20:2307:20 | Unary | r2307_32 | | ir.cpp:2307:20:2307:20 | Unary | r2307_38 | | ir.cpp:2307:20:2307:20 | Unary | r2307_41 | +| ir.cpp:2307:20:2307:20 | Unary | r2307_52 | | ir.cpp:2307:20:2307:20 | Unary | r2307_54 | -| ir.cpp:2307:20:2307:20 | Unary | r2307_56 | -| ir.cpp:2307:20:2307:20 | Unary | r2307_78 | +| ir.cpp:2307:20:2307:20 | Unary | r2307_74 | | ir.cpp:2307:20:2307:55 | Address | &:r2307_2 | | ir.cpp:2307:20:2307:55 | Address | &:r2307_2 | | ir.cpp:2307:20:2307:55 | Arg(this) | this:r2307_2 | @@ -12250,11 +12185,11 @@ | ir.cpp:2307:20:2307:55 | SideEffect | ~m2307_12 | | ir.cpp:2307:20:2307:55 | StoreValue | r2307_22 | | ir.cpp:2307:20:2307:55 | Unary | r2307_2 | -| ir.cpp:2307:20:2307:56 | Address | &:r2307_61 | -| ir.cpp:2307:20:2307:56 | Arg(0) | 0:r2307_61 | -| ir.cpp:2307:20:2307:56 | SideEffect | ~m2307_64 | -| ir.cpp:2307:20:2307:56 | Unary | r2307_59 | -| ir.cpp:2307:20:2307:56 | Unary | r2307_60 | +| ir.cpp:2307:20:2307:56 | Address | &:r2307_57 | +| ir.cpp:2307:20:2307:56 | Arg(0) | 0:r2307_57 | +| ir.cpp:2307:20:2307:56 | SideEffect | ~m2307_60 | +| ir.cpp:2307:20:2307:56 | Unary | r2307_55 | +| ir.cpp:2307:20:2307:56 | Unary | r2307_56 | | ir.cpp:2307:40:2307:54 | Address | &:r2307_5 | | ir.cpp:2307:40:2307:54 | Address | &:r2307_5 | | ir.cpp:2307:40:2307:54 | Address | &:r2307_5 | @@ -12277,9 +12212,9 @@ | ir.cpp:2308:16:2308:17 | CallTarget | func:r2308_3 | | ir.cpp:2308:16:2308:17 | ChiPartial | partial:m2308_5 | | ir.cpp:2308:16:2308:17 | ChiPartial | partial:m2308_7 | -| ir.cpp:2308:16:2308:17 | ChiTotal | total:m2307_64 | +| ir.cpp:2308:16:2308:17 | ChiTotal | total:m2307_60 | | ir.cpp:2308:16:2308:17 | ChiTotal | total:m2308_2 | -| ir.cpp:2308:16:2308:17 | SideEffect | ~m2307_64 | +| ir.cpp:2308:16:2308:17 | SideEffect | ~m2307_60 | | ir.cpp:2309:5:2309:5 | Address | &:r2309_1 | | ir.cpp:2309:5:2309:5 | Address | &:r2309_1 | | ir.cpp:2309:5:2309:5 | Arg(this) | this:r2309_1 | @@ -12310,9 +12245,9 @@ | ir.cpp:2311:18:2311:25 | CallTarget | func:r2311_3 | | ir.cpp:2311:18:2311:25 | ChiPartial | partial:m2311_7 | | ir.cpp:2311:18:2311:25 | ChiPartial | partial:m2311_10 | -| ir.cpp:2311:18:2311:25 | ChiTotal | total:m2307_49 | +| ir.cpp:2311:18:2311:25 | ChiTotal | total:m2307_44 | | ir.cpp:2311:18:2311:25 | ChiTotal | total:m2311_2 | -| ir.cpp:2311:18:2311:25 | SideEffect | ~m2307_49 | +| ir.cpp:2311:18:2311:25 | SideEffect | ~m2307_44 | | ir.cpp:2311:28:2311:29 | Address | &:r2311_12 | | ir.cpp:2311:28:2311:29 | Address | &:r2311_12 | | ir.cpp:2311:28:2311:29 | Address | &:r2311_42 | @@ -12710,7 +12645,7 @@ | ir.cpp:2407:12:2407:21 | StoreValue | r2407_2 | | ir.cpp:2410:6:2410:40 | ChiPartial | partial:m2410_3 | | ir.cpp:2410:6:2410:40 | ChiTotal | total:m2410_2 | -| ir.cpp:2410:6:2410:40 | SideEffect | ~m2430_56 | +| ir.cpp:2410:6:2410:40 | SideEffect | ~m2430_51 | | ir.cpp:2411:9:2411:46 | Address | &:r2411_1 | | ir.cpp:2411:9:2411:46 | Condition | r2411_22 | | ir.cpp:2411:14:2411:14 | Address | &:r2411_18 | @@ -12891,56 +12826,47 @@ | ir.cpp:2430:40:2430:44 | ChiTotal | total:m2430_7 | | ir.cpp:2430:40:2430:44 | SideEffect | ~m2430_7 | | ir.cpp:2430:40:2430:44 | StoreValue | r2430_11 | -| ir.cpp:2430:54:2430:54 | Address | &:r2430_58 | +| ir.cpp:2430:54:2430:54 | Address | &:r2430_56 | | ir.cpp:2430:58:2430:58 | Address | &:r2430_32 | | ir.cpp:2430:58:2430:58 | Address | &:r2430_38 | -| ir.cpp:2430:58:2430:58 | Address | &:r2430_61 | -| ir.cpp:2430:58:2430:58 | Address | &:r2430_66 | -| ir.cpp:2430:58:2430:58 | Address | &:r2430_66 | +| ir.cpp:2430:58:2430:58 | Address | &:r2430_59 | +| ir.cpp:2430:58:2430:58 | Address | &:r2430_62 | +| ir.cpp:2430:58:2430:58 | Address | &:r2430_62 | | ir.cpp:2430:58:2430:58 | Arg(this) | this:r0_2 | | ir.cpp:2430:58:2430:58 | Arg(this) | this:r0_5 | | ir.cpp:2430:58:2430:58 | Arg(this) | this:r0_7 | | ir.cpp:2430:58:2430:58 | Arg(this) | this:r0_8 | | ir.cpp:2430:58:2430:58 | Arg(this) | this:r0_15 | -| ir.cpp:2430:58:2430:58 | Arg(this) | this:r2430_66 | +| ir.cpp:2430:58:2430:58 | Arg(this) | this:r2430_62 | | ir.cpp:2430:58:2430:58 | CallTarget | func:r2430_34 | | ir.cpp:2430:58:2430:58 | CallTarget | func:r2430_40 | | ir.cpp:2430:58:2430:58 | CallTarget | func:r2430_46 | | ir.cpp:2430:58:2430:58 | CallTarget | func:r2430_47 | -| ir.cpp:2430:58:2430:58 | CallTarget | func:r2430_60 | -| ir.cpp:2430:58:2430:58 | CallTarget | func:r2430_67 | +| ir.cpp:2430:58:2430:58 | CallTarget | func:r2430_58 | +| ir.cpp:2430:58:2430:58 | CallTarget | func:r2430_63 | | ir.cpp:2430:58:2430:58 | ChiPartial | partial:m2430_50 | | ir.cpp:2430:58:2430:58 | ChiPartial | partial:m2430_52 | -| ir.cpp:2430:58:2430:58 | ChiPartial | partial:m2430_55 | -| ir.cpp:2430:58:2430:58 | ChiPartial | partial:m2430_62 | -| ir.cpp:2430:58:2430:58 | ChiPartial | partial:m2430_69 | -| ir.cpp:2430:58:2430:58 | ChiPartial | partial:m2430_72 | +| ir.cpp:2430:58:2430:58 | ChiPartial | partial:m2430_66 | | ir.cpp:2430:58:2430:58 | ChiTotal | total:m0_9 | | ir.cpp:2430:58:2430:58 | ChiTotal | total:m2430_43 | | ir.cpp:2430:58:2430:58 | ChiTotal | total:m2430_44 | -| ir.cpp:2430:58:2430:58 | ChiTotal | total:m2430_51 | -| ir.cpp:2430:58:2430:58 | ChiTotal | total:m2430_56 | -| ir.cpp:2430:58:2430:58 | ChiTotal | total:m2430_63 | | ir.cpp:2430:58:2430:58 | Condition | r2430_54 | | ir.cpp:2430:58:2430:58 | Load | m2430_30 | | ir.cpp:2430:58:2430:58 | Load | m2430_30 | | ir.cpp:2430:58:2430:58 | Phi | from 9:m2430_36 | | ir.cpp:2430:58:2430:58 | Phi | from 9:~m2430_26 | -| ir.cpp:2430:58:2430:58 | Phi | from 11:m2430_73 | -| ir.cpp:2430:58:2430:58 | Phi | from 11:~m2430_70 | +| ir.cpp:2430:58:2430:58 | Phi | from 11:m2430_67 | +| ir.cpp:2430:58:2430:58 | Phi | from 11:~m2430_51 | | ir.cpp:2430:58:2430:58 | SideEffect | m2430_43 | | ir.cpp:2430:58:2430:58 | SideEffect | ~m2430_44 | -| ir.cpp:2430:58:2430:58 | SideEffect | ~m2430_51 | -| ir.cpp:2430:58:2430:58 | SideEffect | ~m2430_56 | -| ir.cpp:2430:58:2430:58 | SideEffect | ~m2430_63 | | ir.cpp:2430:58:2430:58 | StoreValue | r2430_35 | | ir.cpp:2430:58:2430:58 | StoreValue | r2430_41 | | ir.cpp:2430:58:2430:58 | Unary | r2430_33 | | ir.cpp:2430:58:2430:58 | Unary | r2430_39 | | ir.cpp:2430:58:2430:58 | Unary | r2430_45 | | ir.cpp:2430:58:2430:58 | Unary | r2430_48 | -| ir.cpp:2430:58:2430:58 | Unary | r2430_59 | -| ir.cpp:2430:58:2430:58 | Unary | r2430_68 | +| ir.cpp:2430:58:2430:58 | Unary | r2430_57 | +| ir.cpp:2430:58:2430:58 | Unary | r2430_64 | | ir.cpp:2430:58:2430:77 | Address | &:r2430_19 | | ir.cpp:2430:58:2430:77 | Address | &:r2430_19 | | ir.cpp:2430:58:2430:77 | Arg(this) | this:r2430_19 | @@ -12952,14 +12878,14 @@ | ir.cpp:2430:58:2430:77 | SideEffect | ~m2430_13 | | ir.cpp:2430:58:2430:77 | StoreValue | r2430_29 | | ir.cpp:2430:58:2430:77 | Unary | r2430_19 | -| ir.cpp:2430:58:2430:78 | Load | ~m2430_63 | -| ir.cpp:2430:58:2430:78 | StoreValue | r2430_64 | +| ir.cpp:2430:58:2430:78 | Load | ~m2430_51 | +| ir.cpp:2430:58:2430:78 | StoreValue | r2430_60 | | ir.cpp:2430:76:2430:76 | Address | &:r2430_22 | | ir.cpp:2430:76:2430:76 | Arg(0) | 0:r2430_23 | | ir.cpp:2430:76:2430:76 | Load | m2430_17 | | ir.cpp:2431:9:2431:9 | Address | &:r2431_4 | | ir.cpp:2431:9:2431:9 | Address | &:r2431_4 | -| ir.cpp:2431:9:2431:9 | Load | m2430_65 | +| ir.cpp:2431:9:2431:9 | Load | m2430_61 | | ir.cpp:2431:9:2431:9 | Unary | r2431_5 | | ir.cpp:2431:9:2431:14 | Left | r2431_6 | | ir.cpp:2431:9:2431:14 | StoreValue | r2431_8 | diff --git a/cpp/ql/test/library-tests/ir/ir/raw_ir.expected b/cpp/ql/test/library-tests/ir/ir/raw_ir.expected index 5cd8ff2cd4a..e3687fd91e0 100644 --- a/cpp/ql/test/library-tests/ir/ir/raw_ir.expected +++ b/cpp/ql/test/library-tests/ir/ir/raw_ir.expected @@ -6910,22 +6910,20 @@ ir.cpp: # 1127| mu1127_25(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r0_6 #-----| r0_11(iterator) = Load[#temp0:0] : &:r0_6, ~m? # 1127| r1127_26(bool) = Call[operator!=] : func:r1127_20, this:r0_5, 0:r0_11 -# 1127| mu1127_27(unknown) = ^CallSideEffect : ~m? #-----| v0_12(void) = ^IndirectReadSideEffect[-1] : &:r0_5, ~m? -# 1127| v1127_28(void) = ConditionalBranch : r1127_26 +# 1127| v1127_27(void) = ConditionalBranch : r1127_26 #-----| False -> Block 5 #-----| True -> Block 2 # 1127| Block 2 -# 1127| r1127_29(glval) = VariableAddress[e] : -# 1127| r1127_30(glval>) = VariableAddress[(__begin)] : -#-----| r0_13(glval>) = Convert : r1127_30 -# 1127| r1127_31(glval) = FunctionAddress[operator*] : -# 1127| r1127_32(int &) = Call[operator*] : func:r1127_31, this:r0_13 -# 1127| mu1127_33(unknown) = ^CallSideEffect : ~m? +# 1127| r1127_28(glval) = VariableAddress[e] : +# 1127| r1127_29(glval>) = VariableAddress[(__begin)] : +#-----| r0_13(glval>) = Convert : r1127_29 +# 1127| r1127_30(glval) = FunctionAddress[operator*] : +# 1127| r1127_31(int &) = Call[operator*] : func:r1127_30, this:r0_13 #-----| v0_14(void) = ^IndirectReadSideEffect[-1] : &:r0_13, ~m? -# 1127| r1127_34(int) = Load[?] : &:r1127_32, ~m? -# 1127| mu1127_35(int) = Store[e] : &:r1127_29, r1127_34 +# 1127| r1127_32(int) = Load[?] : &:r1127_31, ~m? +# 1127| mu1127_33(int) = Store[e] : &:r1127_28, r1127_32 # 1128| r1128_1(glval) = VariableAddress[e] : # 1128| r1128_2(int) = Load[e] : &:r1128_1, ~m? # 1128| r1128_3(int) = Constant[0] : @@ -6939,14 +6937,13 @@ ir.cpp: #-----| Goto -> Block 4 # 1127| Block 4 -# 1127| v1127_36(void) = NoOp : -# 1127| r1127_37(glval>) = VariableAddress[(__begin)] : -# 1127| r1127_38(glval) = FunctionAddress[operator++] : -# 1127| r1127_39(iterator &) = Call[operator++] : func:r1127_38, this:r1127_37 -# 1127| mu1127_40(unknown) = ^CallSideEffect : ~m? -# 1127| v1127_41(void) = ^IndirectReadSideEffect[-1] : &:r1127_37, ~m? -# 1127| mu1127_42(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r1127_37 -# 1127| r1127_43(glval>) = CopyValue : r1127_39 +# 1127| v1127_34(void) = NoOp : +# 1127| r1127_35(glval>) = VariableAddress[(__begin)] : +# 1127| r1127_36(glval) = FunctionAddress[operator++] : +# 1127| r1127_37(iterator &) = Call[operator++] : func:r1127_36, this:r1127_35 +# 1127| v1127_38(void) = ^IndirectReadSideEffect[-1] : &:r1127_35, ~m? +# 1127| mu1127_39(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r1127_35 +# 1127| r1127_40(glval>) = CopyValue : r1127_37 #-----| Goto (back edge) -> Block 1 # 1133| Block 5 @@ -6990,34 +6987,31 @@ ir.cpp: # 1133| mu1133_25(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r0_20 #-----| r0_25(iterator) = Load[#temp0:0] : &:r0_20, ~m? # 1133| r1133_26(bool) = Call[operator!=] : func:r1133_20, this:r0_19, 0:r0_25 -# 1133| mu1133_27(unknown) = ^CallSideEffect : ~m? #-----| v0_26(void) = ^IndirectReadSideEffect[-1] : &:r0_19, ~m? -# 1133| v1133_28(void) = ConditionalBranch : r1133_26 +# 1133| v1133_27(void) = ConditionalBranch : r1133_26 #-----| False -> Block 10 #-----| True -> Block 8 # 1133| Block 7 -# 1133| r1133_29(glval>) = VariableAddress[(__begin)] : -# 1133| r1133_30(glval) = FunctionAddress[operator++] : -# 1133| r1133_31(iterator &) = Call[operator++] : func:r1133_30, this:r1133_29 -# 1133| mu1133_32(unknown) = ^CallSideEffect : ~m? -# 1133| v1133_33(void) = ^IndirectReadSideEffect[-1] : &:r1133_29, ~m? -# 1133| mu1133_34(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r1133_29 -# 1133| r1133_35(glval>) = CopyValue : r1133_31 +# 1133| r1133_28(glval>) = VariableAddress[(__begin)] : +# 1133| r1133_29(glval) = FunctionAddress[operator++] : +# 1133| r1133_30(iterator &) = Call[operator++] : func:r1133_29, this:r1133_28 +# 1133| v1133_31(void) = ^IndirectReadSideEffect[-1] : &:r1133_28, ~m? +# 1133| mu1133_32(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r1133_28 +# 1133| r1133_33(glval>) = CopyValue : r1133_30 #-----| Goto (back edge) -> Block 6 # 1133| Block 8 -# 1133| r1133_36(glval) = VariableAddress[e] : -# 1133| r1133_37(glval>) = VariableAddress[(__begin)] : -#-----| r0_27(glval>) = Convert : r1133_37 -# 1133| r1133_38(glval) = FunctionAddress[operator*] : -# 1133| r1133_39(int &) = Call[operator*] : func:r1133_38, this:r0_27 -# 1133| mu1133_40(unknown) = ^CallSideEffect : ~m? +# 1133| r1133_34(glval) = VariableAddress[e] : +# 1133| r1133_35(glval>) = VariableAddress[(__begin)] : +#-----| r0_27(glval>) = Convert : r1133_35 +# 1133| r1133_36(glval) = FunctionAddress[operator*] : +# 1133| r1133_37(int &) = Call[operator*] : func:r1133_36, this:r0_27 #-----| v0_28(void) = ^IndirectReadSideEffect[-1] : &:r0_27, ~m? -# 1133| r1133_41(glval) = CopyValue : r1133_39 -# 1133| r1133_42(glval) = Convert : r1133_41 -# 1133| r1133_43(int &) = CopyValue : r1133_42 -# 1133| mu1133_44(int &) = Store[e] : &:r1133_36, r1133_43 +# 1133| r1133_38(glval) = CopyValue : r1133_37 +# 1133| r1133_39(glval) = Convert : r1133_38 +# 1133| r1133_40(int &) = CopyValue : r1133_39 +# 1133| mu1133_41(int &) = Store[e] : &:r1133_34, r1133_40 # 1134| r1134_1(glval) = VariableAddress[e] : # 1134| r1134_2(int &) = Load[e] : &:r1134_1, ~m? # 1134| r1134_3(int) = Load[?] : &:r1134_2, ~m? @@ -12620,22 +12614,20 @@ ir.cpp: # 2215| mu2215_34(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r0_8 #-----| r0_13(iterator) = Load[#temp0:0] : &:r0_8, ~m? # 2215| r2215_35(bool) = Call[operator!=] : func:r2215_29, this:r0_7, 0:r0_13 -# 2215| mu2215_36(unknown) = ^CallSideEffect : ~m? #-----| v0_14(void) = ^IndirectReadSideEffect[-1] : &:r0_7, ~m? -# 2215| v2215_37(void) = ConditionalBranch : r2215_35 +# 2215| v2215_36(void) = ConditionalBranch : r2215_35 #-----| False -> Block 14 #-----| True -> Block 12 # 2215| Block 12 -# 2215| r2215_38(glval) = VariableAddress[y] : -# 2215| r2215_39(glval>) = VariableAddress[(__begin)] : -#-----| r0_15(glval>) = Convert : r2215_39 -# 2215| r2215_40(glval) = FunctionAddress[operator*] : -# 2215| r2215_41(ClassWithDestructor &) = Call[operator*] : func:r2215_40, this:r0_15 -# 2215| mu2215_42(unknown) = ^CallSideEffect : ~m? +# 2215| r2215_37(glval) = VariableAddress[y] : +# 2215| r2215_38(glval>) = VariableAddress[(__begin)] : +#-----| r0_15(glval>) = Convert : r2215_38 +# 2215| r2215_39(glval) = FunctionAddress[operator*] : +# 2215| r2215_40(ClassWithDestructor &) = Call[operator*] : func:r2215_39, this:r0_15 #-----| v0_16(void) = ^IndirectReadSideEffect[-1] : &:r0_15, ~m? -# 2215| r2215_43(ClassWithDestructor) = Load[?] : &:r2215_41, ~m? -# 2215| mu2215_44(ClassWithDestructor) = Store[y] : &:r2215_38, r2215_43 +# 2215| r2215_41(ClassWithDestructor) = Load[?] : &:r2215_40, ~m? +# 2215| mu2215_42(ClassWithDestructor) = Store[y] : &:r2215_37, r2215_41 # 2216| r2216_1(glval) = VariableAddress[y] : # 2216| r2216_2(glval) = FunctionAddress[set_x] : # 2216| r2216_3(char) = Constant[97] : @@ -12643,28 +12635,27 @@ ir.cpp: # 2216| mu2216_5(unknown) = ^CallSideEffect : ~m? # 2216| v2216_6(void) = ^IndirectReadSideEffect[-1] : &:r2216_1, ~m? # 2216| mu2216_7(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2216_1 -# 2215| r2215_45(glval) = VariableAddress[y] : -# 2215| r2215_46(glval) = FunctionAddress[~ClassWithDestructor] : -# 2215| v2215_47(void) = Call[~ClassWithDestructor] : func:r2215_46, this:r2215_45 -# 2215| mu2215_48(unknown) = ^CallSideEffect : ~m? -# 2215| v2215_49(void) = ^IndirectReadSideEffect[-1] : &:r2215_45, ~m? -# 2215| mu2215_50(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2215_45 -# 2215| r2215_51(glval>) = VariableAddress[(__begin)] : -# 2215| r2215_52(glval) = FunctionAddress[operator++] : -# 2215| r2215_53(iterator &) = Call[operator++] : func:r2215_52, this:r2215_51 -# 2215| mu2215_54(unknown) = ^CallSideEffect : ~m? -# 2215| v2215_55(void) = ^IndirectReadSideEffect[-1] : &:r2215_51, ~m? -# 2215| mu2215_56(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2215_51 -# 2215| r2215_57(glval>) = CopyValue : r2215_53 +# 2215| r2215_43(glval) = VariableAddress[y] : +# 2215| r2215_44(glval) = FunctionAddress[~ClassWithDestructor] : +# 2215| v2215_45(void) = Call[~ClassWithDestructor] : func:r2215_44, this:r2215_43 +# 2215| mu2215_46(unknown) = ^CallSideEffect : ~m? +# 2215| v2215_47(void) = ^IndirectReadSideEffect[-1] : &:r2215_43, ~m? +# 2215| mu2215_48(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2215_43 +# 2215| r2215_49(glval>) = VariableAddress[(__begin)] : +# 2215| r2215_50(glval) = FunctionAddress[operator++] : +# 2215| r2215_51(iterator &) = Call[operator++] : func:r2215_50, this:r2215_49 +# 2215| v2215_52(void) = ^IndirectReadSideEffect[-1] : &:r2215_49, ~m? +# 2215| mu2215_53(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2215_49 +# 2215| r2215_54(glval>) = CopyValue : r2215_51 #-----| Goto (back edge) -> Block 11 # 2215| Block 13 -# 2215| r2215_58(glval>) = VariableAddress[ys] : -# 2215| r2215_59(glval) = FunctionAddress[~vector] : -# 2215| v2215_60(void) = Call[~vector] : func:r2215_59, this:r2215_58 -# 2215| mu2215_61(unknown) = ^CallSideEffect : ~m? -# 2215| v2215_62(void) = ^IndirectReadSideEffect[-1] : &:r2215_58, ~m? -# 2215| mu2215_63(vector) = ^IndirectMayWriteSideEffect[-1] : &:r2215_58 +# 2215| r2215_55(glval>) = VariableAddress[ys] : +# 2215| r2215_56(glval) = FunctionAddress[~vector] : +# 2215| v2215_57(void) = Call[~vector] : func:r2215_56, this:r2215_55 +# 2215| mu2215_58(unknown) = ^CallSideEffect : ~m? +# 2215| v2215_59(void) = ^IndirectReadSideEffect[-1] : &:r2215_55, ~m? +# 2215| mu2215_60(vector) = ^IndirectMayWriteSideEffect[-1] : &:r2215_55 #-----| Goto -> Block 14 # 2218| Block 14 @@ -12719,22 +12710,20 @@ ir.cpp: # 2218| mu2218_34(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r0_24 #-----| r0_29(iterator) = Load[#temp0:0] : &:r0_24, ~m? # 2218| r2218_35(bool) = Call[operator!=] : func:r2218_29, this:r0_23, 0:r0_29 -# 2218| mu2218_36(unknown) = ^CallSideEffect : ~m? #-----| v0_30(void) = ^IndirectReadSideEffect[-1] : &:r0_23, ~m? -# 2218| v2218_37(void) = ConditionalBranch : r2218_35 +# 2218| v2218_36(void) = ConditionalBranch : r2218_35 #-----| False -> Block 20 #-----| True -> Block 16 # 2218| Block 16 -# 2218| r2218_38(glval) = VariableAddress[y] : -# 2218| r2218_39(glval>) = VariableAddress[(__begin)] : -#-----| r0_31(glval>) = Convert : r2218_39 -# 2218| r2218_40(glval) = FunctionAddress[operator*] : -# 2218| r2218_41(ClassWithDestructor &) = Call[operator*] : func:r2218_40, this:r0_31 -# 2218| mu2218_42(unknown) = ^CallSideEffect : ~m? +# 2218| r2218_37(glval) = VariableAddress[y] : +# 2218| r2218_38(glval>) = VariableAddress[(__begin)] : +#-----| r0_31(glval>) = Convert : r2218_38 +# 2218| r2218_39(glval) = FunctionAddress[operator*] : +# 2218| r2218_40(ClassWithDestructor &) = Call[operator*] : func:r2218_39, this:r0_31 #-----| v0_32(void) = ^IndirectReadSideEffect[-1] : &:r0_31, ~m? -# 2218| r2218_43(ClassWithDestructor) = Load[?] : &:r2218_41, ~m? -# 2218| mu2218_44(ClassWithDestructor) = Store[y] : &:r2218_38, r2218_43 +# 2218| r2218_41(ClassWithDestructor) = Load[?] : &:r2218_40, ~m? +# 2218| mu2218_42(ClassWithDestructor) = Store[y] : &:r2218_37, r2218_41 # 2219| r2219_1(glval) = VariableAddress[y] : # 2219| r2219_2(glval) = FunctionAddress[set_x] : # 2219| r2219_3(char) = Constant[97] : @@ -12757,18 +12746,18 @@ ir.cpp: # 2221| Block 17 # 2221| v2221_1(void) = NoOp : -# 2218| r2218_45(glval) = VariableAddress[y] : -# 2218| r2218_46(glval) = FunctionAddress[~ClassWithDestructor] : -# 2218| v2218_47(void) = Call[~ClassWithDestructor] : func:r2218_46, this:r2218_45 -# 2218| mu2218_48(unknown) = ^CallSideEffect : ~m? -# 2218| v2218_49(void) = ^IndirectReadSideEffect[-1] : &:r2218_45, ~m? -# 2218| mu2218_50(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2218_45 -# 2218| r2218_51(glval>) = VariableAddress[ys] : -# 2218| r2218_52(glval) = FunctionAddress[~vector] : -# 2218| v2218_53(void) = Call[~vector] : func:r2218_52, this:r2218_51 -# 2218| mu2218_54(unknown) = ^CallSideEffect : ~m? -# 2218| v2218_55(void) = ^IndirectReadSideEffect[-1] : &:r2218_51, ~m? -# 2218| mu2218_56(vector) = ^IndirectMayWriteSideEffect[-1] : &:r2218_51 +# 2218| r2218_43(glval) = VariableAddress[y] : +# 2218| r2218_44(glval) = FunctionAddress[~ClassWithDestructor] : +# 2218| v2218_45(void) = Call[~ClassWithDestructor] : func:r2218_44, this:r2218_43 +# 2218| mu2218_46(unknown) = ^CallSideEffect : ~m? +# 2218| v2218_47(void) = ^IndirectReadSideEffect[-1] : &:r2218_43, ~m? +# 2218| mu2218_48(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2218_43 +# 2218| r2218_49(glval>) = VariableAddress[ys] : +# 2218| r2218_50(glval) = FunctionAddress[~vector] : +# 2218| v2218_51(void) = Call[~vector] : func:r2218_50, this:r2218_49 +# 2218| mu2218_52(unknown) = ^CallSideEffect : ~m? +# 2218| v2218_53(void) = ^IndirectReadSideEffect[-1] : &:r2218_49, ~m? +# 2218| mu2218_54(vector) = ^IndirectMayWriteSideEffect[-1] : &:r2218_49 # 2233| r2233_1(glval) = VariableAddress[x] : # 2233| r2233_2(glval) = FunctionAddress[~ClassWithDestructor] : # 2233| v2233_3(void) = Call[~ClassWithDestructor] : func:r2233_2, this:r2233_1 @@ -12778,28 +12767,27 @@ ir.cpp: #-----| Goto -> Block 1 # 2218| Block 18 -# 2218| r2218_57(glval) = VariableAddress[y] : -# 2218| r2218_58(glval) = FunctionAddress[~ClassWithDestructor] : -# 2218| v2218_59(void) = Call[~ClassWithDestructor] : func:r2218_58, this:r2218_57 -# 2218| mu2218_60(unknown) = ^CallSideEffect : ~m? -# 2218| v2218_61(void) = ^IndirectReadSideEffect[-1] : &:r2218_57, ~m? -# 2218| mu2218_62(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2218_57 -# 2218| r2218_63(glval>) = VariableAddress[(__begin)] : -# 2218| r2218_64(glval) = FunctionAddress[operator++] : -# 2218| r2218_65(iterator &) = Call[operator++] : func:r2218_64, this:r2218_63 -# 2218| mu2218_66(unknown) = ^CallSideEffect : ~m? -# 2218| v2218_67(void) = ^IndirectReadSideEffect[-1] : &:r2218_63, ~m? -# 2218| mu2218_68(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2218_63 -# 2218| r2218_69(glval>) = CopyValue : r2218_65 +# 2218| r2218_55(glval) = VariableAddress[y] : +# 2218| r2218_56(glval) = FunctionAddress[~ClassWithDestructor] : +# 2218| v2218_57(void) = Call[~ClassWithDestructor] : func:r2218_56, this:r2218_55 +# 2218| mu2218_58(unknown) = ^CallSideEffect : ~m? +# 2218| v2218_59(void) = ^IndirectReadSideEffect[-1] : &:r2218_55, ~m? +# 2218| mu2218_60(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2218_55 +# 2218| r2218_61(glval>) = VariableAddress[(__begin)] : +# 2218| r2218_62(glval) = FunctionAddress[operator++] : +# 2218| r2218_63(iterator &) = Call[operator++] : func:r2218_62, this:r2218_61 +# 2218| v2218_64(void) = ^IndirectReadSideEffect[-1] : &:r2218_61, ~m? +# 2218| mu2218_65(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2218_61 +# 2218| r2218_66(glval>) = CopyValue : r2218_63 #-----| Goto (back edge) -> Block 15 # 2218| Block 19 -# 2218| r2218_70(glval>) = VariableAddress[ys] : -# 2218| r2218_71(glval) = FunctionAddress[~vector] : -# 2218| v2218_72(void) = Call[~vector] : func:r2218_71, this:r2218_70 -# 2218| mu2218_73(unknown) = ^CallSideEffect : ~m? -# 2218| v2218_74(void) = ^IndirectReadSideEffect[-1] : &:r2218_70, ~m? -# 2218| mu2218_75(vector) = ^IndirectMayWriteSideEffect[-1] : &:r2218_70 +# 2218| r2218_67(glval>) = VariableAddress[ys] : +# 2218| r2218_68(glval) = FunctionAddress[~vector] : +# 2218| v2218_69(void) = Call[~vector] : func:r2218_68, this:r2218_67 +# 2218| mu2218_70(unknown) = ^CallSideEffect : ~m? +# 2218| v2218_71(void) = ^IndirectReadSideEffect[-1] : &:r2218_67, ~m? +# 2218| mu2218_72(vector) = ^IndirectMayWriteSideEffect[-1] : &:r2218_67 #-----| Goto -> Block 20 # 2224| Block 20 @@ -12850,32 +12838,29 @@ ir.cpp: # 2224| mu2224_30(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r0_40 #-----| r0_45(iterator) = Load[#temp0:0] : &:r0_40, ~m? # 2224| r2224_31(bool) = Call[operator!=] : func:r2224_25, this:r0_39, 0:r0_45 -# 2224| mu2224_32(unknown) = ^CallSideEffect : ~m? #-----| v0_46(void) = ^IndirectReadSideEffect[-1] : &:r0_39, ~m? -# 2224| v2224_33(void) = ConditionalBranch : r2224_31 +# 2224| v2224_32(void) = ConditionalBranch : r2224_31 #-----| False -> Block 26 #-----| True -> Block 23 # 2224| Block 22 -# 2224| r2224_34(glval>) = VariableAddress[(__begin)] : -# 2224| r2224_35(glval) = FunctionAddress[operator++] : -# 2224| r2224_36(iterator &) = Call[operator++] : func:r2224_35, this:r2224_34 -# 2224| mu2224_37(unknown) = ^CallSideEffect : ~m? -# 2224| v2224_38(void) = ^IndirectReadSideEffect[-1] : &:r2224_34, ~m? -# 2224| mu2224_39(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2224_34 -# 2224| r2224_40(glval>) = CopyValue : r2224_36 +# 2224| r2224_33(glval>) = VariableAddress[(__begin)] : +# 2224| r2224_34(glval) = FunctionAddress[operator++] : +# 2224| r2224_35(iterator &) = Call[operator++] : func:r2224_34, this:r2224_33 +# 2224| v2224_36(void) = ^IndirectReadSideEffect[-1] : &:r2224_33, ~m? +# 2224| mu2224_37(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2224_33 +# 2224| r2224_38(glval>) = CopyValue : r2224_35 #-----| Goto (back edge) -> Block 21 # 2224| Block 23 -# 2224| r2224_41(glval) = VariableAddress[y] : -# 2224| r2224_42(glval>) = VariableAddress[(__begin)] : -#-----| r0_47(glval>) = Convert : r2224_42 -# 2224| r2224_43(glval) = FunctionAddress[operator*] : -# 2224| r2224_44(int &) = Call[operator*] : func:r2224_43, this:r0_47 -# 2224| mu2224_45(unknown) = ^CallSideEffect : ~m? +# 2224| r2224_39(glval) = VariableAddress[y] : +# 2224| r2224_40(glval>) = VariableAddress[(__begin)] : +#-----| r0_47(glval>) = Convert : r2224_40 +# 2224| r2224_41(glval) = FunctionAddress[operator*] : +# 2224| r2224_42(int &) = Call[operator*] : func:r2224_41, this:r0_47 #-----| v0_48(void) = ^IndirectReadSideEffect[-1] : &:r0_47, ~m? -# 2224| r2224_46(int) = Load[?] : &:r2224_44, ~m? -# 2224| mu2224_47(int) = Store[y] : &:r2224_41, r2224_46 +# 2224| r2224_43(int) = Load[?] : &:r2224_42, ~m? +# 2224| mu2224_44(int) = Store[y] : &:r2224_39, r2224_43 # 2225| r2225_1(glval) = VariableAddress[y] : # 2225| r2225_2(int) = Load[y] : &:r2225_1, ~m? # 2225| r2225_3(int) = Constant[1] : @@ -12886,12 +12871,12 @@ ir.cpp: # 2226| Block 24 # 2226| v2226_1(void) = NoOp : -# 2224| r2224_48(glval>) = VariableAddress[ys] : -# 2224| r2224_49(glval) = FunctionAddress[~vector] : -# 2224| v2224_50(void) = Call[~vector] : func:r2224_49, this:r2224_48 -# 2224| mu2224_51(unknown) = ^CallSideEffect : ~m? -# 2224| v2224_52(void) = ^IndirectReadSideEffect[-1] : &:r2224_48, ~m? -# 2224| mu2224_53(vector) = ^IndirectMayWriteSideEffect[-1] : &:r2224_48 +# 2224| r2224_45(glval>) = VariableAddress[ys] : +# 2224| r2224_46(glval) = FunctionAddress[~vector] : +# 2224| v2224_47(void) = Call[~vector] : func:r2224_46, this:r2224_45 +# 2224| mu2224_48(unknown) = ^CallSideEffect : ~m? +# 2224| v2224_49(void) = ^IndirectReadSideEffect[-1] : &:r2224_45, ~m? +# 2224| mu2224_50(vector) = ^IndirectMayWriteSideEffect[-1] : &:r2224_45 # 2233| r2233_7(glval) = VariableAddress[x] : # 2233| r2233_8(glval) = FunctionAddress[~ClassWithDestructor] : # 2233| v2233_9(void) = Call[~ClassWithDestructor] : func:r2233_8, this:r2233_7 @@ -12901,12 +12886,12 @@ ir.cpp: #-----| Goto -> Block 1 # 2224| Block 25 -# 2224| r2224_54(glval>) = VariableAddress[ys] : -# 2224| r2224_55(glval) = FunctionAddress[~vector] : -# 2224| v2224_56(void) = Call[~vector] : func:r2224_55, this:r2224_54 -# 2224| mu2224_57(unknown) = ^CallSideEffect : ~m? -# 2224| v2224_58(void) = ^IndirectReadSideEffect[-1] : &:r2224_54, ~m? -# 2224| mu2224_59(vector) = ^IndirectMayWriteSideEffect[-1] : &:r2224_54 +# 2224| r2224_51(glval>) = VariableAddress[ys] : +# 2224| r2224_52(glval) = FunctionAddress[~vector] : +# 2224| v2224_53(void) = Call[~vector] : func:r2224_52, this:r2224_51 +# 2224| mu2224_54(unknown) = ^CallSideEffect : ~m? +# 2224| v2224_55(void) = ^IndirectReadSideEffect[-1] : &:r2224_51, ~m? +# 2224| mu2224_56(vector) = ^IndirectMayWriteSideEffect[-1] : &:r2224_51 #-----| Goto -> Block 26 # 2229| Block 26 @@ -12961,22 +12946,20 @@ ir.cpp: # 2229| mu2229_34(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r0_56 #-----| r0_61(iterator) = Load[#temp0:0] : &:r0_56, ~m? # 2229| r2229_35(bool) = Call[operator!=] : func:r2229_29, this:r0_55, 0:r0_61 -# 2229| mu2229_36(unknown) = ^CallSideEffect : ~m? #-----| v0_62(void) = ^IndirectReadSideEffect[-1] : &:r0_55, ~m? -# 2229| v2229_37(void) = ConditionalBranch : r2229_35 +# 2229| v2229_36(void) = ConditionalBranch : r2229_35 #-----| False -> Block 30 #-----| True -> Block 28 # 2229| Block 28 -# 2229| r2229_38(glval) = VariableAddress[y] : -# 2229| r2229_39(glval>) = VariableAddress[(__begin)] : -#-----| r0_63(glval>) = Convert : r2229_39 -# 2229| r2229_40(glval) = FunctionAddress[operator*] : -# 2229| r2229_41(ClassWithDestructor &) = Call[operator*] : func:r2229_40, this:r0_63 -# 2229| mu2229_42(unknown) = ^CallSideEffect : ~m? +# 2229| r2229_37(glval) = VariableAddress[y] : +# 2229| r2229_38(glval>) = VariableAddress[(__begin)] : +#-----| r0_63(glval>) = Convert : r2229_38 +# 2229| r2229_39(glval) = FunctionAddress[operator*] : +# 2229| r2229_40(ClassWithDestructor &) = Call[operator*] : func:r2229_39, this:r0_63 #-----| v0_64(void) = ^IndirectReadSideEffect[-1] : &:r0_63, ~m? -# 2229| r2229_43(ClassWithDestructor) = Load[?] : &:r2229_41, ~m? -# 2229| mu2229_44(ClassWithDestructor) = Store[y] : &:r2229_38, r2229_43 +# 2229| r2229_41(ClassWithDestructor) = Load[?] : &:r2229_40, ~m? +# 2229| mu2229_42(ClassWithDestructor) = Store[y] : &:r2229_37, r2229_41 # 2230| r2230_1(glval) = VariableAddress[z1] : # 2230| mu2230_2(ClassWithDestructor) = Uninitialized[z1] : &:r2230_1 # 2230| r2230_3(glval) = FunctionAddress[ClassWithDestructor] : @@ -13001,28 +12984,27 @@ ir.cpp: # 2232| mu2232_10(unknown) = ^CallSideEffect : ~m? # 2232| v2232_11(void) = ^IndirectReadSideEffect[-1] : &:r2232_7, ~m? # 2232| mu2232_12(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2232_7 -# 2229| r2229_45(glval) = VariableAddress[y] : -# 2229| r2229_46(glval) = FunctionAddress[~ClassWithDestructor] : -# 2229| v2229_47(void) = Call[~ClassWithDestructor] : func:r2229_46, this:r2229_45 -# 2229| mu2229_48(unknown) = ^CallSideEffect : ~m? -# 2229| v2229_49(void) = ^IndirectReadSideEffect[-1] : &:r2229_45, ~m? -# 2229| mu2229_50(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2229_45 -# 2229| r2229_51(glval>) = VariableAddress[(__begin)] : -# 2229| r2229_52(glval) = FunctionAddress[operator++] : -# 2229| r2229_53(iterator &) = Call[operator++] : func:r2229_52, this:r2229_51 -# 2229| mu2229_54(unknown) = ^CallSideEffect : ~m? -# 2229| v2229_55(void) = ^IndirectReadSideEffect[-1] : &:r2229_51, ~m? -# 2229| mu2229_56(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2229_51 -# 2229| r2229_57(glval>) = CopyValue : r2229_53 +# 2229| r2229_43(glval) = VariableAddress[y] : +# 2229| r2229_44(glval) = FunctionAddress[~ClassWithDestructor] : +# 2229| v2229_45(void) = Call[~ClassWithDestructor] : func:r2229_44, this:r2229_43 +# 2229| mu2229_46(unknown) = ^CallSideEffect : ~m? +# 2229| v2229_47(void) = ^IndirectReadSideEffect[-1] : &:r2229_43, ~m? +# 2229| mu2229_48(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2229_43 +# 2229| r2229_49(glval>) = VariableAddress[(__begin)] : +# 2229| r2229_50(glval) = FunctionAddress[operator++] : +# 2229| r2229_51(iterator &) = Call[operator++] : func:r2229_50, this:r2229_49 +# 2229| v2229_52(void) = ^IndirectReadSideEffect[-1] : &:r2229_49, ~m? +# 2229| mu2229_53(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2229_49 +# 2229| r2229_54(glval>) = CopyValue : r2229_51 #-----| Goto (back edge) -> Block 27 # 2229| Block 29 -# 2229| r2229_58(glval>) = VariableAddress[ys] : -# 2229| r2229_59(glval) = FunctionAddress[~vector] : -# 2229| v2229_60(void) = Call[~vector] : func:r2229_59, this:r2229_58 -# 2229| mu2229_61(unknown) = ^CallSideEffect : ~m? -# 2229| v2229_62(void) = ^IndirectReadSideEffect[-1] : &:r2229_58, ~m? -# 2229| mu2229_63(vector) = ^IndirectMayWriteSideEffect[-1] : &:r2229_58 +# 2229| r2229_55(glval>) = VariableAddress[ys] : +# 2229| r2229_56(glval) = FunctionAddress[~vector] : +# 2229| v2229_57(void) = Call[~vector] : func:r2229_56, this:r2229_55 +# 2229| mu2229_58(unknown) = ^CallSideEffect : ~m? +# 2229| v2229_59(void) = ^IndirectReadSideEffect[-1] : &:r2229_55, ~m? +# 2229| mu2229_60(vector) = ^IndirectMayWriteSideEffect[-1] : &:r2229_55 #-----| Goto -> Block 30 # 2233| Block 30 @@ -13610,29 +13592,27 @@ ir.cpp: # 2307| mu2307_38(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r0_8 #-----| r0_13(iterator) = Load[#temp0:0] : &:r0_8, ~m? # 2307| r2307_39(bool) = Call[operator!=] : func:r2307_33, this:r0_7, 0:r0_13 -# 2307| mu2307_40(unknown) = ^CallSideEffect : ~m? #-----| v0_14(void) = ^IndirectReadSideEffect[-1] : &:r0_7, ~m? -# 2307| v2307_41(void) = ConditionalBranch : r2307_39 +# 2307| v2307_40(void) = ConditionalBranch : r2307_39 #-----| False -> Block 6 #-----| True -> Block 5 # 2307| Block 5 -# 2307| r2307_42(glval) = VariableAddress[s] : -# 2307| mu2307_43(String) = Uninitialized[s] : &:r2307_42 -# 2307| r2307_44(glval) = FunctionAddress[String] : -# 2307| r2307_45(glval>) = VariableAddress[(__begin)] : -#-----| r0_15(glval>) = Convert : r2307_45 -# 2307| r2307_46(glval) = FunctionAddress[operator*] : -# 2307| r2307_47(String &) = Call[operator*] : func:r2307_46, this:r0_15 -# 2307| mu2307_48(unknown) = ^CallSideEffect : ~m? +# 2307| r2307_41(glval) = VariableAddress[s] : +# 2307| mu2307_42(String) = Uninitialized[s] : &:r2307_41 +# 2307| r2307_43(glval) = FunctionAddress[String] : +# 2307| r2307_44(glval>) = VariableAddress[(__begin)] : +#-----| r0_15(glval>) = Convert : r2307_44 +# 2307| r2307_45(glval) = FunctionAddress[operator*] : +# 2307| r2307_46(String &) = Call[operator*] : func:r2307_45, this:r0_15 #-----| v0_16(void) = ^IndirectReadSideEffect[-1] : &:r0_15, ~m? -# 2307| r2307_49(glval) = CopyValue : r2307_47 -# 2307| r2307_50(glval) = Convert : r2307_49 -# 2307| r2307_51(String &) = CopyValue : r2307_50 -# 2307| v2307_52(void) = Call[String] : func:r2307_44, this:r2307_42, 0:r2307_51 -# 2307| mu2307_53(unknown) = ^CallSideEffect : ~m? -# 2307| v2307_54(void) = ^BufferReadSideEffect[0] : &:r2307_51, ~m? -# 2307| mu2307_55(String) = ^IndirectMayWriteSideEffect[-1] : &:r2307_42 +# 2307| r2307_47(glval) = CopyValue : r2307_46 +# 2307| r2307_48(glval) = Convert : r2307_47 +# 2307| r2307_49(String &) = CopyValue : r2307_48 +# 2307| v2307_50(void) = Call[String] : func:r2307_43, this:r2307_41, 0:r2307_49 +# 2307| mu2307_51(unknown) = ^CallSideEffect : ~m? +# 2307| v2307_52(void) = ^BufferReadSideEffect[0] : &:r2307_49, ~m? +# 2307| mu2307_53(String) = ^IndirectMayWriteSideEffect[-1] : &:r2307_41 # 2308| r2308_1(glval) = VariableAddress[s2] : # 2308| mu2308_2(String) = Uninitialized[s2] : &:r2308_1 # 2308| r2308_3(glval) = FunctionAddress[String] : @@ -13645,19 +13625,18 @@ ir.cpp: # 2309| mu2309_4(unknown) = ^CallSideEffect : ~m? # 2309| v2309_5(void) = ^IndirectReadSideEffect[-1] : &:r2309_1, ~m? # 2309| mu2309_6(String) = ^IndirectMayWriteSideEffect[-1] : &:r2309_1 -# 2307| r2307_56(glval) = VariableAddress[s] : -# 2307| r2307_57(glval) = FunctionAddress[~String] : -# 2307| v2307_58(void) = Call[~String] : func:r2307_57, this:r2307_56 -# 2307| mu2307_59(unknown) = ^CallSideEffect : ~m? -# 2307| v2307_60(void) = ^IndirectReadSideEffect[-1] : &:r2307_56, ~m? -# 2307| mu2307_61(String) = ^IndirectMayWriteSideEffect[-1] : &:r2307_56 -# 2307| r2307_62(glval>) = VariableAddress[(__begin)] : -# 2307| r2307_63(glval) = FunctionAddress[operator++] : -# 2307| r2307_64(iterator &) = Call[operator++] : func:r2307_63, this:r2307_62 -# 2307| mu2307_65(unknown) = ^CallSideEffect : ~m? -# 2307| v2307_66(void) = ^IndirectReadSideEffect[-1] : &:r2307_62, ~m? -# 2307| mu2307_67(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2307_62 -# 2307| r2307_68(glval>) = CopyValue : r2307_64 +# 2307| r2307_54(glval) = VariableAddress[s] : +# 2307| r2307_55(glval) = FunctionAddress[~String] : +# 2307| v2307_56(void) = Call[~String] : func:r2307_55, this:r2307_54 +# 2307| mu2307_57(unknown) = ^CallSideEffect : ~m? +# 2307| v2307_58(void) = ^IndirectReadSideEffect[-1] : &:r2307_54, ~m? +# 2307| mu2307_59(String) = ^IndirectMayWriteSideEffect[-1] : &:r2307_54 +# 2307| r2307_60(glval>) = VariableAddress[(__begin)] : +# 2307| r2307_61(glval) = FunctionAddress[operator++] : +# 2307| r2307_62(iterator &) = Call[operator++] : func:r2307_61, this:r2307_60 +# 2307| v2307_63(void) = ^IndirectReadSideEffect[-1] : &:r2307_60, ~m? +# 2307| mu2307_64(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2307_60 +# 2307| r2307_65(glval>) = CopyValue : r2307_62 #-----| Goto (back edge) -> Block 4 # 2311| Block 6 @@ -14322,22 +14301,20 @@ ir.cpp: # 2430| mu2430_43(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r0_8 #-----| r0_13(iterator) = Load[#temp0:0] : &:r0_8, ~m? # 2430| r2430_44(bool) = Call[operator!=] : func:r2430_38, this:r0_7, 0:r0_13 -# 2430| mu2430_45(unknown) = ^CallSideEffect : ~m? #-----| v0_14(void) = ^IndirectReadSideEffect[-1] : &:r0_7, ~m? -# 2430| v2430_46(void) = ConditionalBranch : r2430_44 +# 2430| v2430_45(void) = ConditionalBranch : r2430_44 #-----| False -> Block 13 #-----| True -> Block 12 # 2430| Block 12 -# 2430| r2430_47(glval) = VariableAddress[y] : -# 2430| r2430_48(glval>) = VariableAddress[(__begin)] : -#-----| r0_15(glval>) = Convert : r2430_48 -# 2430| r2430_49(glval) = FunctionAddress[operator*] : -# 2430| r2430_50(char &) = Call[operator*] : func:r2430_49, this:r0_15 -# 2430| mu2430_51(unknown) = ^CallSideEffect : ~m? +# 2430| r2430_46(glval) = VariableAddress[y] : +# 2430| r2430_47(glval>) = VariableAddress[(__begin)] : +#-----| r0_15(glval>) = Convert : r2430_47 +# 2430| r2430_48(glval) = FunctionAddress[operator*] : +# 2430| r2430_49(char &) = Call[operator*] : func:r2430_48, this:r0_15 #-----| v0_16(void) = ^IndirectReadSideEffect[-1] : &:r0_15, ~m? -# 2430| r2430_52(char) = Load[?] : &:r2430_50, ~m? -# 2430| mu2430_53(char) = Store[y] : &:r2430_47, r2430_52 +# 2430| r2430_50(char) = Load[?] : &:r2430_49, ~m? +# 2430| mu2430_51(char) = Store[y] : &:r2430_46, r2430_50 # 2431| r2431_1(glval) = VariableAddress[x] : # 2431| r2431_2(char) = Load[x] : &:r2431_1, ~m? # 2431| r2431_3(int) = Convert : r2431_2 @@ -14347,13 +14324,12 @@ ir.cpp: # 2431| r2431_7(int) = Add : r2431_6, r2431_3 # 2431| r2431_8(char) = Convert : r2431_7 # 2431| mu2431_9(char) = Store[y] : &:r2431_4, r2431_8 -# 2430| r2430_54(glval>) = VariableAddress[(__begin)] : -# 2430| r2430_55(glval) = FunctionAddress[operator++] : -# 2430| r2430_56(iterator &) = Call[operator++] : func:r2430_55, this:r2430_54 -# 2430| mu2430_57(unknown) = ^CallSideEffect : ~m? -# 2430| v2430_58(void) = ^IndirectReadSideEffect[-1] : &:r2430_54, ~m? -# 2430| mu2430_59(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2430_54 -# 2430| r2430_60(glval>) = CopyValue : r2430_56 +# 2430| r2430_52(glval>) = VariableAddress[(__begin)] : +# 2430| r2430_53(glval) = FunctionAddress[operator++] : +# 2430| r2430_54(iterator &) = Call[operator++] : func:r2430_53, this:r2430_52 +# 2430| v2430_55(void) = ^IndirectReadSideEffect[-1] : &:r2430_52, ~m? +# 2430| mu2430_56(iterator) = ^IndirectMayWriteSideEffect[-1] : &:r2430_52 +# 2430| r2430_57(glval>) = CopyValue : r2430_54 #-----| Goto (back edge) -> Block 11 # 2432| Block 13 From 1ae22d078128dc776414c74d993364153c3848c0 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Wed, 10 Apr 2024 14:55:29 +0100 Subject: [PATCH 94/97] C++: Improve comment based on PR feedback. --- .../cpp/models/implementations/Iterator.qll | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/cpp/ql/lib/semmle/code/cpp/models/implementations/Iterator.qll b/cpp/ql/lib/semmle/code/cpp/models/implementations/Iterator.qll index f8d31b5583f..9034f3123a6 100644 --- a/cpp/ql/lib/semmle/code/cpp/models/implementations/Iterator.qll +++ b/cpp/ql/lib/semmle/code/cpp/models/implementations/Iterator.qll @@ -141,10 +141,12 @@ private class IteratorCrementNonMemberOperatorModel extends IteratorCrementNonMe } override predicate hasSpecificWriteSideEffect(ParameterIndex i, boolean buffer, boolean mustWrite) { - // `buffer` must be `true` or `mustWrite` must be `false` to ensure that - // the IR alias analysis doesn't think that `it++` doesn't completely override - // the value of the iterator. Ideally, the IR alias analysis shouldn't deal with - // iterators, but this is necessary for taintflow to get any results in our iterator tests. + // We have two choices here: either `buffer` must be `true` or `mustWrite` + // must be `false` to ensure that the IR alias analysis doesn't think that + // `it++` completely override the value of the iterator. + // We choose `mustWrite` must be `false`. In that case, the value of + // `buffer` isn't super important (it just decides which kind of read side + // effect will be emitted). i = 0 and buffer = false and mustWrite = false } @@ -193,10 +195,12 @@ private class IteratorCrementMemberOperatorModel extends IteratorCrementMemberOp } override predicate hasSpecificWriteSideEffect(ParameterIndex i, boolean buffer, boolean mustWrite) { - // `buffer` must be `true` or `mustWrite` must be `false` to ensure that - // the IR alias analysis doesn't think that `it++` doesn't completely override - // the value of the iterator. Ideally, the IR alias analysis shouldn't deal with - // iterators, but this is necessary for taintflow to get any results in our iterator tests. + // We have two choices here: either `buffer` must be `true` or `mustWrite` + // must be `false` to ensure that the IR alias analysis doesn't think that + // `it++` completely override the value of the iterator. + // We choose `mustWrite` must be `false`. In that case, the value of + // `buffer` isn't super important (it just decides which kind of read side + // effect will be emitted). i = -1 and buffer = false and mustWrite = false } From 9c9ed13ede5324f6dd37264b0d7e360715e786ee Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Wed, 10 Apr 2024 14:57:40 +0100 Subject: [PATCH 95/97] C++: More comments. --- cpp/ql/lib/semmle/code/cpp/models/implementations/Iterator.qll | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/ql/lib/semmle/code/cpp/models/implementations/Iterator.qll b/cpp/ql/lib/semmle/code/cpp/models/implementations/Iterator.qll index 9034f3123a6..0144b97d2bf 100644 --- a/cpp/ql/lib/semmle/code/cpp/models/implementations/Iterator.qll +++ b/cpp/ql/lib/semmle/code/cpp/models/implementations/Iterator.qll @@ -573,6 +573,8 @@ private class IteratorAssignmentMemberOperatorModel extends IteratorAssignmentMe override predicate hasOnlySpecificWriteSideEffects() { any() } override predicate hasSpecificWriteSideEffect(ParameterIndex i, boolean buffer, boolean mustWrite) { + // See the comment on `IteratorCrementMemberOperatorModel::hasSpecificWriteSideEffect` + // for an explanation of these values. i = -1 and buffer = false and mustWrite = false } From 0ed07310243f6f8ebb66786915c5108b56724f73 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Wed, 10 Apr 2024 15:04:29 +0100 Subject: [PATCH 96/97] C++: Reduce comment duplication. --- .../semmle/code/cpp/models/implementations/Iterator.qll | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cpp/ql/lib/semmle/code/cpp/models/implementations/Iterator.qll b/cpp/ql/lib/semmle/code/cpp/models/implementations/Iterator.qll index 0144b97d2bf..8e527c7e694 100644 --- a/cpp/ql/lib/semmle/code/cpp/models/implementations/Iterator.qll +++ b/cpp/ql/lib/semmle/code/cpp/models/implementations/Iterator.qll @@ -141,12 +141,8 @@ private class IteratorCrementNonMemberOperatorModel extends IteratorCrementNonMe } override predicate hasSpecificWriteSideEffect(ParameterIndex i, boolean buffer, boolean mustWrite) { - // We have two choices here: either `buffer` must be `true` or `mustWrite` - // must be `false` to ensure that the IR alias analysis doesn't think that - // `it++` completely override the value of the iterator. - // We choose `mustWrite` must be `false`. In that case, the value of - // `buffer` isn't super important (it just decides which kind of read side - // effect will be emitted). + // See the comment on `IteratorCrementMemberOperatorModel::hasSpecificWriteSideEffect` + // for an explanation of these values. i = 0 and buffer = false and mustWrite = false } From 05f5879a2c772c1fcbc138bfddb83fcb7c17f8fe Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 10 Apr 2024 16:11:20 +0200 Subject: [PATCH 97/97] Bazel: `swift/third_party/load.bzl` cleanup and `bazel mod tidy` enablement --- MODULE.bazel | 2 ++ swift/third_party/load.bzl | 44 +++++++++++--------------------------- 2 files changed, 14 insertions(+), 32 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index e8c79e8377f..8e1cab32dad 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -31,6 +31,8 @@ pip.parse( use_repo(pip, "codegen_deps") swift_deps = use_extension("//swift/third_party:load.bzl", "swift_deps") + +# following list can be kept in sync with `bazel mod tidy` use_repo( swift_deps, "binlog", diff --git a/swift/third_party/load.bzl b/swift/third_party/load.bzl index 7e55ef50601..b6698520c38 100644 --- a/swift/third_party/load.bzl +++ b/swift/third_party/load.bzl @@ -40,7 +40,7 @@ def _get_toolchain_url(info): info.extension, ) -def _toolchains(repository_name): +def _toolchains(): rules = { "tar.gz": http_archive, "pkg": _pkg_archive, @@ -51,7 +51,7 @@ def _toolchains(repository_name): name = "swift_toolchain_%s" % arch, url = _get_toolchain_url(info), sha256 = info.sha, - build_file = _build(repository_name, "swift-toolchain-%s" % arch), + build_file = _build % "swift-toolchain-%s" % arch, strip_prefix = "%s-%s" % (_swift_version, info.suffix), ) @@ -109,10 +109,9 @@ def _github_archive(*, name, repository, commit, build_file = None, sha256 = Non sha256 = sha256, ) -def _build(repository_name, package): - return "@%s//swift/third_party:BUILD.%s.bazel" % (repository_name, package) +_build = "//swift/third_party:BUILD.%s.bazel" -def load_dependencies(module_ctx = None, repository_name = "codeql"): +def load_dependencies(module_ctx): for repo_arch, arch in _swift_arch_map.items(): sha256 = _swift_sha_map[repo_arch] @@ -122,15 +121,15 @@ def load_dependencies(module_ctx = None, repository_name = "codeql"): _swift_prebuilt_version, repo_arch, ), - build_file = _build(repository_name, "swift-llvm-support"), + build_file = _build % "swift-llvm-support", sha256 = sha256, ) - _toolchains(repository_name) + _toolchains() _github_archive( name = "picosha2", - build_file = _build(repository_name, "picosha2"), + build_file = _build % "picosha2", repository = "okdshin/PicoSHA2", commit = "27fcf6979298949e8a462e16d09a0351c18fcaf2", sha256 = "d6647ca45a8b7bdaf027ecb68d041b22a899a0218b7206dee755c558a2725abb", @@ -138,34 +137,15 @@ def load_dependencies(module_ctx = None, repository_name = "codeql"): _github_archive( name = "binlog", - build_file = _build(repository_name, "binlog"), + build_file = _build % "binlog", repository = "morganstanley/binlog", commit = "3fef8846f5ef98e64211e7982c2ead67e0b185a6", sha256 = "f5c61d90a6eff341bf91771f2f465be391fd85397023e1b391c17214f9cbd045", ) - if module_ctx == None: - # legacy workspace loading, remove when transition is complete - _github_archive( - name = "absl", - repository = "abseil/abseil-cpp", - commit = "d2c5297a3c3948de765100cb7e5cccca1210d23c", - sha256 = "735a9efc673f30b3212bfd57f38d5deb152b543e35cd58b412d1363b15242049", - ) - - _github_archive( - name = "json", - repository = "nlohmann/json", - commit = "6af826d0bdb55e4b69e3ad817576745335f243ca", - sha256 = "702bb0231a5e21c0374230fed86c8ae3d07ee50f34ffd420e7f8249854b7d85b", - ) - - _github_archive( - name = "fmt", - repository = "fmtlib/fmt", - build_file = _build(repository_name, "fmt"), - commit = "a0b8a92e3d1532361c2f7feb63babc5c18d00ef2", - sha256 = "ccf872fd4aa9ab3d030d62cffcb258ca27f021b2023a0244b2cf476f984be955", - ) + return module_ctx.extension_metadata( + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) swift_deps = module_extension(load_dependencies)