From da8cc13506f99b310ea6505b1ee319bb9b22dde8 Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Mon, 11 Mar 2024 09:57:02 +0000 Subject: [PATCH 01/50] go extractor: avoid long string concatenations When we see "a" + "b" + "c" + "d", do not add a row to the constvalues table for the intermiediate strings "ab" and "abc". We still have entries for the string literals ("a", "b", "c", and "d") and the whole string concatenation ("abcd"). --- go/extractor/extractor.go | 142 ++++++++++++++++++++++---------------- 1 file changed, 83 insertions(+), 59 deletions(-) diff --git a/go/extractor/extractor.go b/go/extractor/extractor.go index f2ba68a20f0..97f3ed743fb 100644 --- a/go/extractor/extractor.go +++ b/go/extractor/extractor.go @@ -794,7 +794,7 @@ func extractLocalScope(tw *trap.Writer, scope *types.Scope, parentScopeLabel tra func extractFileNode(tw *trap.Writer, nd *ast.File) { lbl := tw.Labeler.FileLabel() - extractExpr(tw, nd.Name, lbl, 0) + extractExpr(tw, nd.Name, lbl, 0, false) for i, decl := range nd.Decls { extractDecl(tw, decl, lbl, i) @@ -851,7 +851,7 @@ func emitScopeNodeInfo(tw *trap.Writer, nd ast.Node, lbl trap.Label) { } // extractExpr extracts AST information for the given expression and all its subexpressions -func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { +func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int, skipExtractingValue bool) { if expr == nil { return } @@ -900,7 +900,7 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { return } kind = dbscheme.EllipsisExpr.Index() - extractExpr(tw, expr.Elt, lbl, 0) + extractExpr(tw, expr.Elt, lbl, 0, false) case *ast.BasicLit: if expr == nil { return @@ -932,28 +932,28 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { return } kind = dbscheme.FuncLitExpr.Index() - extractExpr(tw, expr.Type, lbl, 0) + extractExpr(tw, expr.Type, lbl, 0, false) extractStmt(tw, expr.Body, lbl, 1) case *ast.CompositeLit: if expr == nil { return } kind = dbscheme.CompositeLitExpr.Index() - extractExpr(tw, expr.Type, lbl, 0) + extractExpr(tw, expr.Type, lbl, 0, false) extractExprs(tw, expr.Elts, lbl, 1, 1) case *ast.ParenExpr: if expr == nil { return } kind = dbscheme.ParenExpr.Index() - extractExpr(tw, expr.X, lbl, 0) + extractExpr(tw, expr.X, lbl, 0, false) case *ast.SelectorExpr: if expr == nil { return } kind = dbscheme.SelectorExpr.Index() - extractExpr(tw, expr.X, lbl, 0) - extractExpr(tw, expr.Sel, lbl, 1) + extractExpr(tw, expr.X, lbl, 0, false) + extractExpr(tw, expr.Sel, lbl, 1, false) case *ast.IndexExpr: if expr == nil { return @@ -974,8 +974,8 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { kind = dbscheme.IndexExpr.Index() } } - extractExpr(tw, expr.X, lbl, 0) - extractExpr(tw, expr.Index, lbl, 1) + extractExpr(tw, expr.X, lbl, 0, false) + extractExpr(tw, expr.Index, lbl, 1, false) case *ast.IndexListExpr: if expr == nil { return @@ -993,30 +993,30 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { kind = dbscheme.GenericTypeInstantiationExpr.Index() } } - extractExpr(tw, expr.X, lbl, 0) + extractExpr(tw, expr.X, lbl, 0, false) extractExprs(tw, expr.Indices, lbl, 1, 1) case *ast.SliceExpr: if expr == nil { return } kind = dbscheme.SliceExpr.Index() - extractExpr(tw, expr.X, lbl, 0) - extractExpr(tw, expr.Low, lbl, 1) - extractExpr(tw, expr.High, lbl, 2) - extractExpr(tw, expr.Max, lbl, 3) + extractExpr(tw, expr.X, lbl, 0, false) + extractExpr(tw, expr.Low, lbl, 1, false) + extractExpr(tw, expr.High, lbl, 2, false) + extractExpr(tw, expr.Max, lbl, 3, false) case *ast.TypeAssertExpr: if expr == nil { return } kind = dbscheme.TypeAssertExpr.Index() - extractExpr(tw, expr.X, lbl, 0) - extractExpr(tw, expr.Type, lbl, 1) + extractExpr(tw, expr.X, lbl, 0, false) + extractExpr(tw, expr.Type, lbl, 1, false) case *ast.CallExpr: if expr == nil { return } kind = dbscheme.CallOrConversionExpr.Index() - extractExpr(tw, expr.Fun, lbl, 0) + extractExpr(tw, expr.Fun, lbl, 0, false) extractExprs(tw, expr.Args, lbl, 1, 1) if expr.Ellipsis.IsValid() { dbscheme.HasEllipsisTable.Emit(tw, lbl) @@ -1026,14 +1026,14 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { return } kind = dbscheme.StarExpr.Index() - extractExpr(tw, expr.X, lbl, 0) + extractExpr(tw, expr.X, lbl, 0, false) case *ast.KeyValueExpr: if expr == nil { return } kind = dbscheme.KeyValueExpr.Index() - extractExpr(tw, expr.Key, lbl, 0) - extractExpr(tw, expr.Value, lbl, 1) + extractExpr(tw, expr.Key, lbl, 0, false) + extractExpr(tw, expr.Value, lbl, 1, false) case *ast.UnaryExpr: if expr == nil { return @@ -1047,7 +1047,7 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { } kind = tp.Index() } - extractExpr(tw, expr.X, lbl, 0) + extractExpr(tw, expr.X, lbl, 0, false) case *ast.BinaryExpr: if expr == nil { return @@ -1062,16 +1062,17 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { log.Fatalf("unsupported binary operator %s", expr.Op) } kind = tp.Index() - extractExpr(tw, expr.X, lbl, 0) - extractExpr(tw, expr.Y, lbl, 1) + skipLeft := skipExtractingValueForLeftOperand(tw, expr) + extractExpr(tw, expr.X, lbl, 0, skipLeft) + extractExpr(tw, expr.Y, lbl, 1, false) } case *ast.ArrayType: if expr == nil { return } kind = dbscheme.ArrayTypeExpr.Index() - extractExpr(tw, expr.Len, lbl, 0) - extractExpr(tw, expr.Elt, lbl, 1) + extractExpr(tw, expr.Len, lbl, 0, false) + extractExpr(tw, expr.Elt, lbl, 1, false) case *ast.StructType: if expr == nil { return @@ -1100,8 +1101,8 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { return } kind = dbscheme.MapTypeExpr.Index() - extractExpr(tw, expr.Key, lbl, 0) - extractExpr(tw, expr.Value, lbl, 1) + extractExpr(tw, expr.Key, lbl, 0, false) + extractExpr(tw, expr.Value, lbl, 1, false) case *ast.ChanType: if expr == nil { return @@ -1111,13 +1112,15 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { log.Fatalf("unsupported channel direction %v", expr.Dir) } kind = tp.Index() - extractExpr(tw, expr.Value, lbl, 0) + extractExpr(tw, expr.Value, lbl, 0, false) default: log.Fatalf("unknown expression of type %T", expr) } dbscheme.ExprsTable.Emit(tw, lbl, kind, parent, idx) extractNodeLocation(tw, expr, lbl) - extractValueOf(tw, expr, lbl) + if !skipExtractingValue { + extractValueOf(tw, expr, lbl) + } } // extractExprs extracts AST information for a list of expressions, which are children of @@ -1128,7 +1131,7 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { func extractExprs(tw *trap.Writer, exprs []ast.Expr, parent trap.Label, idx int, dir int) { if exprs != nil { for _, expr := range exprs { - extractExpr(tw, expr, parent, idx) + extractExpr(tw, expr, parent, idx, false) idx += dir } } @@ -1194,11 +1197,11 @@ func extractFields(tw *trap.Writer, fields *ast.FieldList, parent trap.Label, id extractNodeLocation(tw, field, lbl) if field.Names != nil { for i, name := range field.Names { - extractExpr(tw, name, lbl, i+1) + extractExpr(tw, name, lbl, i+1, false) } } - extractExpr(tw, field.Type, lbl, 0) - extractExpr(tw, field.Tag, lbl, -1) + extractExpr(tw, field.Type, lbl, 0, false) + extractExpr(tw, field.Tag, lbl, -1, false) extractDoc(tw, field.Doc, lbl) idx += dir } @@ -1229,21 +1232,21 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { return } kind = dbscheme.LabeledStmtType.Index() - extractExpr(tw, stmt.Label, lbl, 0) + extractExpr(tw, stmt.Label, lbl, 0, false) extractStmt(tw, stmt.Stmt, lbl, 1) case *ast.ExprStmt: if stmt == nil { return } kind = dbscheme.ExprStmtType.Index() - extractExpr(tw, stmt.X, lbl, 0) + extractExpr(tw, stmt.X, lbl, 0, false) case *ast.SendStmt: if stmt == nil { return } kind = dbscheme.SendStmtType.Index() - extractExpr(tw, stmt.Chan, lbl, 0) - extractExpr(tw, stmt.Value, lbl, 1) + extractExpr(tw, stmt.Chan, lbl, 0, false) + extractExpr(tw, stmt.Value, lbl, 1, false) case *ast.IncDecStmt: if stmt == nil { return @@ -1255,7 +1258,7 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { } else { log.Fatalf("unsupported increment/decrement operator %v", stmt.Tok) } - extractExpr(tw, stmt.X, lbl, 0) + extractExpr(tw, stmt.X, lbl, 0, false) case *ast.AssignStmt: if stmt == nil { return @@ -1272,13 +1275,13 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { return } kind = dbscheme.GoStmtType.Index() - extractExpr(tw, stmt.Call, lbl, 0) + extractExpr(tw, stmt.Call, lbl, 0, false) case *ast.DeferStmt: if stmt == nil { return } kind = dbscheme.DeferStmtType.Index() - extractExpr(tw, stmt.Call, lbl, 0) + extractExpr(tw, stmt.Call, lbl, 0, false) case *ast.ReturnStmt: kind = dbscheme.ReturnStmtType.Index() extractExprs(tw, stmt.Results, lbl, 0, 1) @@ -1298,7 +1301,7 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { default: log.Fatalf("unsupported branch statement type %v", stmt.Tok) } - extractExpr(tw, stmt.Label, lbl, 0) + extractExpr(tw, stmt.Label, lbl, 0, false) case *ast.BlockStmt: if stmt == nil { return @@ -1312,7 +1315,7 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { } kind = dbscheme.IfStmtType.Index() extractStmt(tw, stmt.Init, lbl, 0) - extractExpr(tw, stmt.Cond, lbl, 1) + extractExpr(tw, stmt.Cond, lbl, 1, false) extractStmt(tw, stmt.Body, lbl, 2) extractStmt(tw, stmt.Else, lbl, 3) emitScopeNodeInfo(tw, stmt, lbl) @@ -1330,7 +1333,7 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { } kind = dbscheme.ExprSwitchStmtType.Index() extractStmt(tw, stmt.Init, lbl, 0) - extractExpr(tw, stmt.Tag, lbl, 1) + extractExpr(tw, stmt.Tag, lbl, 1, false) extractStmt(tw, stmt.Body, lbl, 2) emitScopeNodeInfo(tw, stmt, lbl) case *ast.TypeSwitchStmt: @@ -1359,7 +1362,7 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { } kind = dbscheme.ForStmtType.Index() extractStmt(tw, stmt.Init, lbl, 0) - extractExpr(tw, stmt.Cond, lbl, 1) + extractExpr(tw, stmt.Cond, lbl, 1, false) extractStmt(tw, stmt.Post, lbl, 2) extractStmt(tw, stmt.Body, lbl, 3) emitScopeNodeInfo(tw, stmt, lbl) @@ -1368,9 +1371,9 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { return } kind = dbscheme.RangeStmtType.Index() - extractExpr(tw, stmt.Key, lbl, 0) - extractExpr(tw, stmt.Value, lbl, 1) - extractExpr(tw, stmt.X, lbl, 2) + extractExpr(tw, stmt.Key, lbl, 0, false) + extractExpr(tw, stmt.Value, lbl, 1, false) + extractExpr(tw, stmt.X, lbl, 2, false) extractStmt(tw, stmt.Body, lbl, 3) emitScopeNodeInfo(tw, stmt, lbl) default: @@ -1428,8 +1431,8 @@ func extractDecl(tw *trap.Writer, decl ast.Decl, parent trap.Label, idx int) { } kind = dbscheme.FuncDeclType.Index() extractFields(tw, decl.Recv, lbl, -1, -1) - extractExpr(tw, decl.Name, lbl, 0) - extractExpr(tw, decl.Type, lbl, 1) + extractExpr(tw, decl.Name, lbl, 0, false) + extractExpr(tw, decl.Type, lbl, 1, false) extractStmt(tw, decl.Body, lbl, 2) extractDoc(tw, decl.Doc, lbl) extractTypeParamDecls(tw, decl.Type.TypeParams, lbl) @@ -1455,8 +1458,8 @@ func extractSpec(tw *trap.Writer, spec ast.Spec, parent trap.Label, idx int) { return } kind = dbscheme.ImportSpecType.Index() - extractExpr(tw, spec.Name, lbl, 0) - extractExpr(tw, spec.Path, lbl, 1) + extractExpr(tw, spec.Name, lbl, 0, false) + extractExpr(tw, spec.Path, lbl, 1, false) extractDoc(tw, spec.Doc, lbl) case *ast.ValueSpec: if spec == nil { @@ -1464,9 +1467,9 @@ func extractSpec(tw *trap.Writer, spec ast.Spec, parent trap.Label, idx int) { } kind = dbscheme.ValueSpecType.Index() for i, name := range spec.Names { - extractExpr(tw, name, lbl, -(1 + i)) + extractExpr(tw, name, lbl, -(1 + i), false) } - extractExpr(tw, spec.Type, lbl, 0) + extractExpr(tw, spec.Type, lbl, 0, false) extractExprs(tw, spec.Values, lbl, 1, 1) extractDoc(tw, spec.Doc, lbl) case *ast.TypeSpec: @@ -1478,9 +1481,9 @@ func extractSpec(tw *trap.Writer, spec ast.Spec, parent trap.Label, idx int) { } else { kind = dbscheme.TypeDefSpecType.Index() } - extractExpr(tw, spec.Name, lbl, 0) + extractExpr(tw, spec.Name, lbl, 0, false) extractTypeParamDecls(tw, spec.TypeParams, lbl) - extractExpr(tw, spec.Type, lbl, 1) + extractExpr(tw, spec.Type, lbl, 1, false) extractDoc(tw, spec.Doc, lbl) } dbscheme.SpecsTable.Emit(tw, lbl, kind, parent, idx) @@ -1909,7 +1912,7 @@ func flattenBinaryExprTree(tw *trap.Writer, e ast.Expr, parent trap.Label, idx i idx = flattenBinaryExprTree(tw, binaryexpr.X, parent, idx) idx = flattenBinaryExprTree(tw, binaryexpr.Y, parent, idx) } else { - extractExpr(tw, e, parent, idx) + extractExpr(tw, e, parent, idx, false) idx = idx + 1 } return idx @@ -1931,10 +1934,10 @@ func extractTypeParamDecls(tw *trap.Writer, fields *ast.FieldList, parent trap.L extractNodeLocation(tw, field, lbl) if field.Names != nil { for i, name := range field.Names { - extractExpr(tw, name, lbl, i+1) + extractExpr(tw, name, lbl, i+1, false) } } - extractExpr(tw, field.Type, lbl, 0) + extractExpr(tw, field.Type, lbl, 0, false) extractDoc(tw, field.Doc, lbl) idx += 1 } @@ -2023,3 +2026,24 @@ func setTypeParamParent(tp *types.TypeParam, newobj types.Object) { log.Fatalf("Parent of type parameter '%s %s' being set to a different value: '%s' vs '%s'", tp.String(), tp.Constraint().String(), obj, newobj) } } + +// skipExtractingValueForLeftOperand returns true if the left operand of `be` +// should not have its value extracted because it is an intermediate value in a +// string concatenation - specifically that the right operand is a string +// literal +func skipExtractingValueForLeftOperand(tw *trap.Writer, be *ast.BinaryExpr) bool { + // check `be` has string type + tpVal := tw.Package.TypesInfo.Types[be] + if tpVal.Value == nil || tpVal.Value.Kind() != constant.String { + return false + } + // check that the right operand of `be` is a basic literal + if _, isBasicLit := be.Y.(*ast.BasicLit); !isBasicLit { + return false + } + // check that the left operand of `be` is not a basic literal + if _, isBasicLit := be.X.(*ast.BasicLit); isBasicLit { + return false + } + return true +} From 530c76ca8b49cf4338cc72124e3b49af28376fd2 Mon Sep 17 00:00:00 2001 From: Kevin Stubbings Date: Wed, 14 Dec 2022 20:59:47 -0800 Subject: [PATCH 02/50] Add New Sanitizers and Modify Old Ones --- .../go/security/TaintedPathCustomizations.qll | 46 ++++++++++++++++++- .../Security/CWE-022/TaintedPath.expected | 22 +++++++-- .../Security/CWE-022/TaintedPath.go | 29 +++++++++++- 3 files changed, 91 insertions(+), 6 deletions(-) diff --git a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll index 42edd470da2..3c0288493f5 100644 --- a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll +++ b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll @@ -74,19 +74,46 @@ module TaintedPath { } /** - * A call to `[file]path.Clean("/" + e)`, considered to sanitize `e` against path traversal. + * A call to `filepath.Clean("/" + e)`, considered to sanitize `e` against path traversal. */ class FilepathCleanSanitizer extends Sanitizer { FilepathCleanSanitizer() { exists(DataFlow::CallNode cleanCall, StringOps::Concatenation concatNode | cleanCall = - any(Function f | f.hasQualifiedName(["path", "path/filepath"], "Clean")).getACall() and + any(Function f | f.hasQualifiedName("path/filepath", "Clean")).getACall() and concatNode = cleanCall.getArgument(0) and concatNode.getOperand(0).asExpr().(StringLit).getValue() = "/" and this = cleanCall.getResult() ) } } + /** + * A call to `filepath.Base(e)`, considered to sanitize `e` against path traversal. + */ + class FilepathBaseSanitizer extends Sanitizer { + FilepathBaseSanitizer() { + exists(Function f, FunctionOutput outp | + f.hasQualifiedName("path/filepath", "Base") and + outp.isResult(0) and + this = outp.getNode(f.getACall()) + ) + } + } + + /** An call to ParseMultipartForm creates multipart.Form and cleans mutlpart.Form.FileHeader.Filename using path.Base() */ + class MultipartClean extends Sanitizer { + MultipartClean() { + exists(DataFlow::FieldReadNode frn, ControlFlow::Node node, DataFlow::CallNode cleanCall, Method get | + get.hasQualifiedName("net/http","Request", "ParseMultipartForm") and + cleanCall = get.getACall() and + cleanCall.asInstruction() = node and + frn.getField().hasQualifiedName("mime/multipart", "FileHeader", "Filename") and + node.getASuccessor*() = frn.asInstruction() + | + this = frn.getBase() + ) + } + } /** * A check of the form `!strings.Contains(nd, "..")`, considered as a sanitizer guard for @@ -105,6 +132,21 @@ module TaintedPath { branch = false } } +/** + * A replacement of the form `!strings.ReplaceAll(nd, "..")` or `!strings.ReplaceAll(nd, ".")`, considered as a sanitizer guard for + * path traversal. + */ + class DotDotReplace extends Sanitizer { + DotDotReplace() { + exists(DataFlow::CallNode cleanCall, DataFlow::Node valueNode | + cleanCall = + any(Function f | f.hasQualifiedName("strings", "ReplaceAll")).getACall() and + valueNode = cleanCall.getArgument(1) and + (valueNode.asExpr().(StringLit).getValue() = ".." or valueNode.asExpr().(StringLit).getValue() = ".") and + this = cleanCall.getResult() + ) + } + } /** * A node `nd` guarded by a check that ensures it is contained within some root folder, diff --git a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected index 34d4180a849..38d968a9e3b 100644 --- a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected +++ b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected @@ -1,19 +1,35 @@ edges +<<<<<<< HEAD | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:13:18:13:30 | call to Query | provenance | | | TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:16:29:16:40 | tainted_path | provenance | | | TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:20:57:20:68 | tainted_path | provenance | | | TaintedPath.go:20:57:20:68 | tainted_path | TaintedPath.go:20:28:20:69 | call to Join | provenance | | | tst.go:14:2:14:39 | ... := ...[1] | tst.go:17:41:17:56 | selection of Filename | provenance | | +======= +| TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:16:29:16:40 | tainted_path | +| TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:20:28:20:69 | call to Join | +| TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:67:28:67:57 | call to Clean | +| TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:77:28:77:56 | call to Base | +| tst.go:14:2:14:39 | ... := ...[1] : pointer type | tst.go:17:41:17:56 | selection of Filename | +>>>>>>> a45343fb6c (Add New Sanitizers and Modify Old Ones) nodes | TaintedPath.go:13:18:13:22 | selection of URL | semmle.label | selection of URL | | TaintedPath.go:13:18:13:30 | call to Query | semmle.label | call to Query | | TaintedPath.go:16:29:16:40 | tainted_path | semmle.label | tainted_path | | TaintedPath.go:20:28:20:69 | call to Join | semmle.label | call to Join | +<<<<<<< HEAD | TaintedPath.go:20:57:20:68 | tainted_path | semmle.label | tainted_path | | tst.go:14:2:14:39 | ... := ...[1] | semmle.label | ... := ...[1] | +======= +| TaintedPath.go:67:28:67:57 | call to Clean | semmle.label | call to Clean | +| TaintedPath.go:77:28:77:56 | call to Base | semmle.label | call to Base | +| tst.go:14:2:14:39 | ... := ...[1] : pointer type | semmle.label | ... := ...[1] : pointer type | +>>>>>>> a45343fb6c (Add New Sanitizers and Modify Old Ones) | tst.go:17:41:17:56 | selection of Filename | semmle.label | selection of Filename | subpaths #select -| TaintedPath.go:16:29:16:40 | tainted_path | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:16:29:16:40 | tainted_path | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | -| TaintedPath.go:20:28:20:69 | call to Join | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:20:28:20:69 | call to Join | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | -| tst.go:17:41:17:56 | selection of Filename | tst.go:14:2:14:39 | ... := ...[1] | tst.go:17:41:17:56 | selection of Filename | This path depends on a $@. | tst.go:14:2:14:39 | ... := ...[1] | user-provided value | +| TaintedPath.go:16:29:16:40 | tainted_path | TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:16:29:16:40 | tainted_path | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | +| TaintedPath.go:20:28:20:69 | call to Join | TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:20:28:20:69 | call to Join | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | +| TaintedPath.go:67:28:67:57 | call to Clean | TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:67:28:67:57 | call to Clean | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | +| TaintedPath.go:77:28:77:56 | call to Base | TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:77:28:77:56 | call to Base | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | +| tst.go:17:41:17:56 | selection of Filename | tst.go:14:2:14:39 | ... := ...[1] : pointer type | tst.go:17:41:17:56 | selection of Filename | This path depends on a $@. | tst.go:14:2:14:39 | ... := ...[1] | user-provided value | diff --git a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go index e663fc3b11d..629b7556e9b 100644 --- a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go +++ b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go @@ -31,6 +31,10 @@ func handler(w http.ResponseWriter, r *http.Request) { w.Write(data) } + // GOOD: Sanitized by strings.ReplaceAll and replaces all .. with empty string + data, _ = ioutil.ReadFile(strings.ReplaceAll(tainted_path, "..", "")) + w.Write(data) + // GOOD: This can only read inside the provided safe path _, err := filepath.Rel("/home/user/safepath", tainted_path) if err == nil { @@ -53,10 +57,33 @@ func handler(w http.ResponseWriter, r *http.Request) { w.Write(data) } - // GOOD: Sanitized by [file]path.Clean with a prepended '/' forcing interpretation + // GOOD: Sanitized by filepath.Clean with a prepended '/' forcing interpretation // as an absolute path, so that Clean will throw away any leading `..` components. data, _ = ioutil.ReadFile(filepath.Clean("/" + tainted_path)) w.Write(data) + + // BAD: Sanitized by path.Clean with a prepended '/' forcing interpretation + // as an absolute path, however is not sufficient for Windows paths. data, _ = ioutil.ReadFile(path.Clean("/" + tainted_path)) w.Write(data) + + // GOOD: Sanitized by filepath.Base with a prepended '/' forcing interpretation + // as an absolute path, so that Base will throw away any leading `..` components. + data, _ = ioutil.ReadFile(filepath.Base("/" + tainted_path)) + w.Write(data) + + // BAD: Sanitized by path.Base with a prepended '/' forcing interpretation + // as an absolute path, however is not sufficient for Windows paths. + data, _ = ioutil.ReadFile(path.Base("/" + tainted_path)) + w.Write(data) + + // GOOD: Multipart.Form.FileHeader.Filename sanitized by filepath.Base when calling ParseMultipartForm + r.ParseMultipartForm(32 << 20) + form := r.MultipartForm + files, ok := form.File["files"] + if !ok { + return + } + data, _ = ioutil.ReadFile(files[0].Filename) + w.Write(data) } From c9b49d3760db97e22ce266e73065c1c1d8b35879 Mon Sep 17 00:00:00 2001 From: Kevin Stubbings Date: Sun, 8 Oct 2023 16:24:25 -0700 Subject: [PATCH 03/50] resolve feedback --- .../go/security/TaintedPathCustomizations.qll | 17 ++++------- .../Security/CWE-022/TaintedPath.expected | 28 +++++++++++-------- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll index 3c0288493f5..815a628fec3 100644 --- a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll +++ b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll @@ -5,7 +5,7 @@ import go import semmle.go.dataflow.barrierguardutil.RegexpCheck - +import DataFlow /** * Provides extension points for customizing the taint tracking configuration for reasoning about * path-traversal vulnerabilities. @@ -100,17 +100,12 @@ module TaintedPath { } } - /** An call to ParseMultipartForm creates multipart.Form and cleans mutlpart.Form.FileHeader.Filename using path.Base() */ + /**An call to ParseMultipartForm creates multipart.Form and cleans mutlpart.Form.FileHeader.Filename using path.Base() */ class MultipartClean extends Sanitizer { MultipartClean() { - exists(DataFlow::FieldReadNode frn, ControlFlow::Node node, DataFlow::CallNode cleanCall, Method get | - get.hasQualifiedName("net/http","Request", "ParseMultipartForm") and - cleanCall = get.getACall() and - cleanCall.asInstruction() = node and + exists(DataFlow::FieldReadNode frn | frn.getField().hasQualifiedName("mime/multipart", "FileHeader", "Filename") and - node.getASuccessor*() = frn.asInstruction() - | - this = frn.getBase() + this = frn ) } } @@ -133,7 +128,7 @@ module TaintedPath { } } /** - * A replacement of the form `!strings.ReplaceAll(nd, "..")` or `!strings.ReplaceAll(nd, ".")`, considered as a sanitizer guard for + * A replacement of the form `!strings.ReplaceAll(nd, "..")` or `!strings.ReplaceAll(nd, ".")`, considered as a sanitizer for * path traversal. */ class DotDotReplace extends Sanitizer { @@ -142,7 +137,7 @@ module TaintedPath { cleanCall = any(Function f | f.hasQualifiedName("strings", "ReplaceAll")).getACall() and valueNode = cleanCall.getArgument(1) and - (valueNode.asExpr().(StringLit).getValue() = ".." or valueNode.asExpr().(StringLit).getValue() = ".") and + valueNode.asExpr().(StringLit).getValue() = ["..", "."] and this = cleanCall.getResult() ) } diff --git a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected index 38d968a9e3b..c3a4fc5a616 100644 --- a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected +++ b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected @@ -1,5 +1,6 @@ edges <<<<<<< HEAD +<<<<<<< HEAD | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:13:18:13:30 | call to Query | provenance | | | TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:16:29:16:40 | tainted_path | provenance | | | TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:20:57:20:68 | tainted_path | provenance | | @@ -12,24 +13,29 @@ edges | TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:77:28:77:56 | call to Base | | tst.go:14:2:14:39 | ... := ...[1] : pointer type | tst.go:17:41:17:56 | selection of Filename | >>>>>>> a45343fb6c (Add New Sanitizers and Modify Old Ones) +======= +| TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:13:18:13:30 | call to Query | +| TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:16:29:16:40 | tainted_path | +| TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:20:57:20:68 | tainted_path | +| TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:67:39:67:56 | ...+... | +| TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:77:38:77:55 | ...+... | +| TaintedPath.go:20:57:20:68 | tainted_path | TaintedPath.go:20:28:20:69 | call to Join | +| TaintedPath.go:67:39:67:56 | ...+... | TaintedPath.go:67:28:67:57 | call to Clean | +| TaintedPath.go:77:38:77:55 | ...+... | TaintedPath.go:77:28:77:56 | call to Base | +>>>>>>> db14838a4f (resolve feedback) nodes | TaintedPath.go:13:18:13:22 | selection of URL | semmle.label | selection of URL | | TaintedPath.go:13:18:13:30 | call to Query | semmle.label | call to Query | | TaintedPath.go:16:29:16:40 | tainted_path | semmle.label | tainted_path | | TaintedPath.go:20:28:20:69 | call to Join | semmle.label | call to Join | -<<<<<<< HEAD | TaintedPath.go:20:57:20:68 | tainted_path | semmle.label | tainted_path | -| tst.go:14:2:14:39 | ... := ...[1] | semmle.label | ... := ...[1] | -======= | TaintedPath.go:67:28:67:57 | call to Clean | semmle.label | call to Clean | +| TaintedPath.go:67:39:67:56 | ...+... | semmle.label | ...+... | | TaintedPath.go:77:28:77:56 | call to Base | semmle.label | call to Base | -| tst.go:14:2:14:39 | ... := ...[1] : pointer type | semmle.label | ... := ...[1] : pointer type | ->>>>>>> a45343fb6c (Add New Sanitizers and Modify Old Ones) -| tst.go:17:41:17:56 | selection of Filename | semmle.label | selection of Filename | +| TaintedPath.go:77:38:77:55 | ...+... | semmle.label | ...+... | subpaths #select -| TaintedPath.go:16:29:16:40 | tainted_path | TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:16:29:16:40 | tainted_path | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | -| TaintedPath.go:20:28:20:69 | call to Join | TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:20:28:20:69 | call to Join | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | -| TaintedPath.go:67:28:67:57 | call to Clean | TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:67:28:67:57 | call to Clean | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | -| TaintedPath.go:77:28:77:56 | call to Base | TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:77:28:77:56 | call to Base | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | -| tst.go:17:41:17:56 | selection of Filename | tst.go:14:2:14:39 | ... := ...[1] : pointer type | tst.go:17:41:17:56 | selection of Filename | This path depends on a $@. | tst.go:14:2:14:39 | ... := ...[1] | user-provided value | +| TaintedPath.go:16:29:16:40 | tainted_path | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:16:29:16:40 | tainted_path | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | +| TaintedPath.go:20:28:20:69 | call to Join | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:20:28:20:69 | call to Join | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | +| TaintedPath.go:67:28:67:57 | call to Clean | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:67:28:67:57 | call to Clean | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | +| TaintedPath.go:77:28:77:56 | call to Base | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:77:28:77:56 | call to Base | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | From c4c0b22bc78075dfa0463f6fc8b09b9e3c1ef34c Mon Sep 17 00:00:00 2001 From: Kevin Stubbings Date: Sun, 8 Oct 2023 16:37:35 -0700 Subject: [PATCH 04/50] Formattinga and change notes --- .../2023-10-08-addional-gopath-sanitizers.md | 5 +++++ .../go/security/TaintedPathCustomizations.qll | 15 ++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 go/ql/lib/change-notes/2023-10-08-addional-gopath-sanitizers.md diff --git a/go/ql/lib/change-notes/2023-10-08-addional-gopath-sanitizers.md b/go/ql/lib/change-notes/2023-10-08-addional-gopath-sanitizers.md new file mode 100644 index 00000000000..0666fee50d5 --- /dev/null +++ b/go/ql/lib/change-notes/2023-10-08-addional-gopath-sanitizers.md @@ -0,0 +1,5 @@ + +* --- +category: minorAnalysis +--- +* Added filepath.Base, strings.ReplaceAll, http.ParseMultipartForm sanitizers and remove path sanitizer. \ No newline at end of file diff --git a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll index 815a628fec3..c6eec12d9ef 100644 --- a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll +++ b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll @@ -6,6 +6,7 @@ import go import semmle.go.dataflow.barrierguardutil.RegexpCheck import DataFlow + /** * Provides extension points for customizing the taint tracking configuration for reasoning about * path-traversal vulnerabilities. @@ -79,15 +80,15 @@ module TaintedPath { class FilepathCleanSanitizer extends Sanitizer { FilepathCleanSanitizer() { exists(DataFlow::CallNode cleanCall, StringOps::Concatenation concatNode | - cleanCall = - any(Function f | f.hasQualifiedName("path/filepath", "Clean")).getACall() and + cleanCall = any(Function f | f.hasQualifiedName("path/filepath", "Clean")).getACall() and concatNode = cleanCall.getArgument(0) and concatNode.getOperand(0).asExpr().(StringLit).getValue() = "/" and this = cleanCall.getResult() ) } } - /** + + /** * A call to `filepath.Base(e)`, considered to sanitize `e` against path traversal. */ class FilepathBaseSanitizer extends Sanitizer { @@ -107,8 +108,8 @@ module TaintedPath { frn.getField().hasQualifiedName("mime/multipart", "FileHeader", "Filename") and this = frn ) - } } + } /** * A check of the form `!strings.Contains(nd, "..")`, considered as a sanitizer guard for @@ -127,15 +128,15 @@ module TaintedPath { branch = false } } -/** + + /** * A replacement of the form `!strings.ReplaceAll(nd, "..")` or `!strings.ReplaceAll(nd, ".")`, considered as a sanitizer for * path traversal. */ class DotDotReplace extends Sanitizer { DotDotReplace() { exists(DataFlow::CallNode cleanCall, DataFlow::Node valueNode | - cleanCall = - any(Function f | f.hasQualifiedName("strings", "ReplaceAll")).getACall() and + cleanCall = any(Function f | f.hasQualifiedName("strings", "ReplaceAll")).getACall() and valueNode = cleanCall.getArgument(1) and valueNode.asExpr().(StringLit).getValue() = ["..", "."] and this = cleanCall.getResult() From 30fe4168e34cc4059c59cabcee77a30e850c7f7c Mon Sep 17 00:00:00 2001 From: Kevin Stubbings Date: Mon, 11 Mar 2024 15:08:10 -0700 Subject: [PATCH 05/50] Removed filepath.base sanitizer --- .../2023-10-08-addional-gopath-sanitizers.md | 5 ---- .../2024-03-11-addional-gopath-sanitizers.md | 5 ++++ .../go/security/TaintedPathCustomizations.qll | 13 ---------- .../Security/CWE-022/TaintedPath.expected | 25 ++----------------- .../Security/CWE-022/TaintedPath.go | 10 -------- 5 files changed, 7 insertions(+), 51 deletions(-) delete mode 100644 go/ql/lib/change-notes/2023-10-08-addional-gopath-sanitizers.md create mode 100644 go/ql/lib/change-notes/2024-03-11-addional-gopath-sanitizers.md diff --git a/go/ql/lib/change-notes/2023-10-08-addional-gopath-sanitizers.md b/go/ql/lib/change-notes/2023-10-08-addional-gopath-sanitizers.md deleted file mode 100644 index 0666fee50d5..00000000000 --- a/go/ql/lib/change-notes/2023-10-08-addional-gopath-sanitizers.md +++ /dev/null @@ -1,5 +0,0 @@ - -* --- -category: minorAnalysis ---- -* Added filepath.Base, strings.ReplaceAll, http.ParseMultipartForm sanitizers and remove path sanitizer. \ No newline at end of file diff --git a/go/ql/lib/change-notes/2024-03-11-addional-gopath-sanitizers.md b/go/ql/lib/change-notes/2024-03-11-addional-gopath-sanitizers.md new file mode 100644 index 00000000000..acfafc26cc5 --- /dev/null +++ b/go/ql/lib/change-notes/2024-03-11-addional-gopath-sanitizers.md @@ -0,0 +1,5 @@ + +* --- +category: minorAnalysis +--- +* Added strings.ReplaceAll, http.ParseMultipartForm sanitizers and remove path sanitizer. \ No newline at end of file diff --git a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll index c6eec12d9ef..70f279d307b 100644 --- a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll +++ b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll @@ -88,19 +88,6 @@ module TaintedPath { } } - /** - * A call to `filepath.Base(e)`, considered to sanitize `e` against path traversal. - */ - class FilepathBaseSanitizer extends Sanitizer { - FilepathBaseSanitizer() { - exists(Function f, FunctionOutput outp | - f.hasQualifiedName("path/filepath", "Base") and - outp.isResult(0) and - this = outp.getNode(f.getACall()) - ) - } - } - /**An call to ParseMultipartForm creates multipart.Form and cleans mutlpart.Form.FileHeader.Filename using path.Base() */ class MultipartClean extends Sanitizer { MultipartClean() { diff --git a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected index c3a4fc5a616..4b4748f0d9c 100644 --- a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected +++ b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected @@ -1,28 +1,10 @@ edges -<<<<<<< HEAD -<<<<<<< HEAD | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:13:18:13:30 | call to Query | provenance | | | TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:16:29:16:40 | tainted_path | provenance | | | TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:20:57:20:68 | tainted_path | provenance | | +| TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:67:39:67:56 | ...+... | provenance | | | TaintedPath.go:20:57:20:68 | tainted_path | TaintedPath.go:20:28:20:69 | call to Join | provenance | | -| tst.go:14:2:14:39 | ... := ...[1] | tst.go:17:41:17:56 | selection of Filename | provenance | | -======= -| TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:16:29:16:40 | tainted_path | -| TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:20:28:20:69 | call to Join | -| TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:67:28:67:57 | call to Clean | -| TaintedPath.go:13:18:13:22 | selection of URL : pointer type | TaintedPath.go:77:28:77:56 | call to Base | -| tst.go:14:2:14:39 | ... := ...[1] : pointer type | tst.go:17:41:17:56 | selection of Filename | ->>>>>>> a45343fb6c (Add New Sanitizers and Modify Old Ones) -======= -| TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:13:18:13:30 | call to Query | -| TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:16:29:16:40 | tainted_path | -| TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:20:57:20:68 | tainted_path | -| TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:67:39:67:56 | ...+... | -| TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:77:38:77:55 | ...+... | -| TaintedPath.go:20:57:20:68 | tainted_path | TaintedPath.go:20:28:20:69 | call to Join | -| TaintedPath.go:67:39:67:56 | ...+... | TaintedPath.go:67:28:67:57 | call to Clean | -| TaintedPath.go:77:38:77:55 | ...+... | TaintedPath.go:77:28:77:56 | call to Base | ->>>>>>> db14838a4f (resolve feedback) +| TaintedPath.go:67:39:67:56 | ...+... | TaintedPath.go:67:28:67:57 | call to Clean | provenance | | nodes | TaintedPath.go:13:18:13:22 | selection of URL | semmle.label | selection of URL | | TaintedPath.go:13:18:13:30 | call to Query | semmle.label | call to Query | @@ -31,11 +13,8 @@ nodes | TaintedPath.go:20:57:20:68 | tainted_path | semmle.label | tainted_path | | TaintedPath.go:67:28:67:57 | call to Clean | semmle.label | call to Clean | | TaintedPath.go:67:39:67:56 | ...+... | semmle.label | ...+... | -| TaintedPath.go:77:28:77:56 | call to Base | semmle.label | call to Base | -| TaintedPath.go:77:38:77:55 | ...+... | semmle.label | ...+... | subpaths #select | TaintedPath.go:16:29:16:40 | tainted_path | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:16:29:16:40 | tainted_path | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | | TaintedPath.go:20:28:20:69 | call to Join | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:20:28:20:69 | call to Join | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | | TaintedPath.go:67:28:67:57 | call to Clean | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:67:28:67:57 | call to Clean | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | -| TaintedPath.go:77:28:77:56 | call to Base | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:77:28:77:56 | call to Base | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | diff --git a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go index 629b7556e9b..378cc3acea5 100644 --- a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go +++ b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go @@ -67,16 +67,6 @@ func handler(w http.ResponseWriter, r *http.Request) { data, _ = ioutil.ReadFile(path.Clean("/" + tainted_path)) w.Write(data) - // GOOD: Sanitized by filepath.Base with a prepended '/' forcing interpretation - // as an absolute path, so that Base will throw away any leading `..` components. - data, _ = ioutil.ReadFile(filepath.Base("/" + tainted_path)) - w.Write(data) - - // BAD: Sanitized by path.Base with a prepended '/' forcing interpretation - // as an absolute path, however is not sufficient for Windows paths. - data, _ = ioutil.ReadFile(path.Base("/" + tainted_path)) - w.Write(data) - // GOOD: Multipart.Form.FileHeader.Filename sanitized by filepath.Base when calling ParseMultipartForm r.ParseMultipartForm(32 << 20) form := r.MultipartForm From 5acc15bfff6c3d9e6bb2e9a6197d34a971fec38a Mon Sep 17 00:00:00 2001 From: Kevin Stubbings Date: Mon, 11 Mar 2024 20:13:08 -0700 Subject: [PATCH 06/50] fix grammar --- go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll index 70f279d307b..5959bc0ffe8 100644 --- a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll +++ b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll @@ -88,7 +88,7 @@ module TaintedPath { } } - /**An call to ParseMultipartForm creates multipart.Form and cleans mutlpart.Form.FileHeader.Filename using path.Base() */ + /**An call to ParseMultipartForm creates multipart.Form and cleans multipart.Form.FileHeader.Filename using path.Base() */ class MultipartClean extends Sanitizer { MultipartClean() { exists(DataFlow::FieldReadNode frn | From 33c17313b413e9f742f825e5bc4c65609f0d1c0a Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Tue, 12 Mar 2024 11:59:10 +0000 Subject: [PATCH 07/50] Add test for not extracting values for intermediate string concatenations --- .../no-intermediate-strings/tst.expected | 11 +++++++++++ .../no-intermediate-strings/tst.go | 5 +++++ .../no-intermediate-strings/tst.ql | 17 +++++++++++++++++ 3 files changed, 33 insertions(+) create mode 100644 go/ql/test/extractor-tests/no-intermediate-strings/tst.expected create mode 100644 go/ql/test/extractor-tests/no-intermediate-strings/tst.go create mode 100644 go/ql/test/extractor-tests/no-intermediate-strings/tst.ql diff --git a/go/ql/test/extractor-tests/no-intermediate-strings/tst.expected b/go/ql/test/extractor-tests/no-intermediate-strings/tst.expected new file mode 100644 index 00000000000..9746d74fb90 --- /dev/null +++ b/go/ql/test/extractor-tests/no-intermediate-strings/tst.expected @@ -0,0 +1,11 @@ +| tst.go:4:6:4:8 | "a" | a | +| tst.go:4:6:4:14 | ...+... | | +| tst.go:4:6:4:20 | ...+... | | +| tst.go:4:6:4:26 | ...+... | | +| tst.go:4:6:4:32 | ...+... | | +| tst.go:4:6:4:38 | ...+... | abcdef | +| tst.go:4:12:4:14 | "b" | b | +| tst.go:4:18:4:20 | "c" | c | +| tst.go:4:24:4:26 | "d" | d | +| tst.go:4:30:4:32 | "e" | e | +| tst.go:4:36:4:38 | "f" | f | diff --git a/go/ql/test/extractor-tests/no-intermediate-strings/tst.go b/go/ql/test/extractor-tests/no-intermediate-strings/tst.go new file mode 100644 index 00000000000..c79c97a8e88 --- /dev/null +++ b/go/ql/test/extractor-tests/no-intermediate-strings/tst.go @@ -0,0 +1,5 @@ +package main + +func main() { + _ = "a" + "b" + "c" + "d" + "e" + "f" +} diff --git a/go/ql/test/extractor-tests/no-intermediate-strings/tst.ql b/go/ql/test/extractor-tests/no-intermediate-strings/tst.ql new file mode 100644 index 00000000000..6367ef51e70 --- /dev/null +++ b/go/ql/test/extractor-tests/no-intermediate-strings/tst.ql @@ -0,0 +1,17 @@ +import go + +string checkStringValue(Expr e) { + result = e.getStringValue() + or + not exists(e.getStringValue()) and result = "" +} + +from Expr e +where e.getType() instanceof StringType +// We should get string values for `"a"`, `"b"`, `"c"` and `"a" + "b" + "c" +// but not `"a" + "b"`. In the extractor we avoid storing the value of +// intermediate strings in string concatenations because in pathological cases +// this could lead to a quadratic blowup in the size of string values stored, +// which then causes performance problems when we iterate through all string +// values. +select e, checkStringValue(e) From 32ebd4eebbcf13613b51c4b8686b3183b0da3c04 Mon Sep 17 00:00:00 2001 From: Max Schaefer Date: Wed, 27 Mar 2024 12:22:24 +0000 Subject: [PATCH 08/50] Automodel: Filter unexploitable types in application mode. We already did this in framework mode. --- .../AutomodelApplicationModeCharacteristics.qll | 16 +++++++++------- .../AutomodelApplicationModeExtraction/Test.java | 11 ++++++----- .../com/github/codeql/test/MyWriter.java | 2 +- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/java/ql/automodel/src/AutomodelApplicationModeCharacteristics.qll b/java/ql/automodel/src/AutomodelApplicationModeCharacteristics.qll index 045886985b8..3023677ece8 100644 --- a/java/ql/automodel/src/AutomodelApplicationModeCharacteristics.qll +++ b/java/ql/automodel/src/AutomodelApplicationModeCharacteristics.qll @@ -28,25 +28,27 @@ newtype TApplicationModeEndpoint = AutomodelJavaUtil::isFromSource(call) and exists(Argument argExpr | arg.asExpr() = argExpr and call = argExpr.getCall() and not argExpr.isVararg() - ) + ) and + not AutomodelJavaUtil::isUnexploitableType(arg.getType()) } or TInstanceArgument(Call call, DataFlow::Node arg) { AutomodelJavaUtil::isFromSource(call) and arg = DataFlow::getInstanceArgument(call) and - not call instanceof ConstructorCall + not call instanceof ConstructorCall and + not AutomodelJavaUtil::isUnexploitableType(arg.getType()) } or TImplicitVarargsArray(Call call, DataFlow::ImplicitVarargsArray arg, int idx) { AutomodelJavaUtil::isFromSource(call) and call = arg.getCall() and - idx = call.getCallee().getVaragsParameterIndex() + idx = call.getCallee().getVaragsParameterIndex() and + not AutomodelJavaUtil::isUnexploitableType(arg.getType()) } or - TMethodReturnValue(Call call) { + TMethodReturnValue(MethodCall call) { AutomodelJavaUtil::isFromSource(call) and - not call instanceof ConstructorCall + not AutomodelJavaUtil::isUnexploitableType(call.getType()) } or TOverriddenParameter(Parameter p, Method overriddenMethod) { AutomodelJavaUtil::isFromSource(p) and - not p.getCallable().callsConstructor(_) and p.getCallable().(Method).overrides(overriddenMethod) } @@ -163,7 +165,7 @@ class ImplicitVarargsArray extends CallArgument, TImplicitVarargsArray { * may be a source. */ class MethodReturnValue extends ApplicationModeEndpoint, TMethodReturnValue { - Call call; + MethodCall call; MethodReturnValue() { this = TMethodReturnValue(call) } diff --git a/java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java b/java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java index 4d6aff63fd0..a8280bcaf27 100644 --- a/java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java +++ b/java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java @@ -19,11 +19,11 @@ class Test { AtomicReference reference = new AtomicReference<>(); // uninteresting (parameterless constructor) reference.set( // $ sinkModelCandidate=set(Object):Argument[this] args[0] // $ negativeSinkExample=set(Object):Argument[0] // modeled as a flow step - ); // $ negativeSourceExample=set(Object):ReturnValue // return type is void + ); // not a source candidate (return type is void) } public static void callSupplier(Supplier supplier) { - supplier.get(); // $ sourceModelCandidate=get():ReturnValue + supplier.get(); // not a source candidate (lambda flow) } public static void copyFiles(Path source, Path target, CopyOption option) throws Exception { @@ -52,7 +52,7 @@ class Test { public static int compareFiles(File f1, File f2) { return f1.compareTo( // $ negativeSinkExample=compareTo(File):Argument[this] f2 // $ negativeSinkExample=compareTo(File):Argument[0] // modeled as not a sink - ); // $ negativeSourceExample=compareTo(File):ReturnValue // return type is int + ); // not a source candidate (return type is int) } public static void FilesWalkExample(Path p, FileVisitOption o) throws Exception { @@ -66,6 +66,7 @@ class Test { public static void WebSocketExample(URLConnection c) throws Exception { c.getInputStream(); // $ sinkModelCandidate=getInputStream():Argument[this] positiveSourceExample=getInputStream():ReturnValue(remote) // not a source candidate (manual modeling) + c.connect(); // $ sinkModelCandidate=connect():Argument[this] // not a source candidate (return type is void) } public static void fileFilterExample(File f, FileFilter ff) { @@ -102,10 +103,10 @@ class MoreTests { Files.delete( p // $ sinkModelCandidate=delete(Path):Argument[0] positiveSinkExample=delete(Path):Argument[0](path-injection) - ); // $ negativeSourceExample=delete(Path):ReturnValue // return type is void + ); // $ not a source candidate (return type is void) Files.deleteIfExists( p // $ sinkModelCandidate=deleteIfExists(Path):Argument[0] positiveSinkExample=deleteIfExists(Path):Argument[0](path-injection) - ); // $ negativeSourceExample=deleteIfExists(Path):ReturnValue // return type is boolean + ); // $ not a source candidate (return type is boolean) } } \ No newline at end of file diff --git a/java/ql/automodel/test/AutomodelFrameworkModeExtraction/com/github/codeql/test/MyWriter.java b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/com/github/codeql/test/MyWriter.java index b31ace21b4d..62bd773cc2e 100644 --- a/java/ql/automodel/test/AutomodelFrameworkModeExtraction/com/github/codeql/test/MyWriter.java +++ b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/com/github/codeql/test/MyWriter.java @@ -2,7 +2,7 @@ package com.github.codeql.test; public class MyWriter extends java.io.Writer { @Override - public void write(char[] cbuf, int off, int len) { // $ sinkModelCandidate=write(char[],int,int):Argument[this] sourceModelCandidate=write(char[],int,int):Parameter[this] sourceModelCandidate=write(char[],int,int):Parameter[0] + public void write(char[] cbuf, int off, int len) { // $ sinkModelCandidate=write(char[],int,int):Argument[this] positiveSinkExample=write(char[],int,int):Argument[0](file-content-store) sourceModelCandidate=write(char[],int,int):Parameter[this] sourceModelCandidate=write(char[],int,int):Parameter[0] } @Override From ce98353d2267558ab05574d2e2c4e2f773e973c8 Mon Sep 17 00:00:00 2001 From: Dave Bartolomeo Date: Tue, 2 Apr 2024 15:15:11 -0400 Subject: [PATCH 09/50] Allow `@param` tags to apply to record parameters --- .../Advisory/Documentation/SpuriousJavadocParam.ql | 12 ++++++++++++ .../src/change-notes/2024-04-02-javadoc-records.md | 5 +++++ .../test/query-tests/SpuriousJavadocParam/Test.java | 13 +++++++++++++ .../test/query-tests/SpuriousJavadocParam/options | 1 + .../query-tests/SpuriousJavadocParam/test.expected | 2 ++ 5 files changed, 33 insertions(+) create mode 100644 java/ql/src/change-notes/2024-04-02-javadoc-records.md create mode 100644 java/ql/test/query-tests/SpuriousJavadocParam/options diff --git a/java/ql/src/Advisory/Documentation/SpuriousJavadocParam.ql b/java/ql/src/Advisory/Documentation/SpuriousJavadocParam.ql index 691811687a7..43afd978f01 100644 --- a/java/ql/src/Advisory/Documentation/SpuriousJavadocParam.ql +++ b/java/ql/src/Advisory/Documentation/SpuriousJavadocParam.ql @@ -32,12 +32,24 @@ where ) or documentable instanceof ClassOrInterface and + not documentable instanceof Record and not exists(TypeVariable tv | tv.getGenericType() = documentable | "<" + tv.getName() + ">" = paramTag.getParamName() ) and msg = "@param tag \"" + paramTag.getParamName() + "\" does not match any actual type parameter of type \"" + documentable.getName() + "\"." + or + documentable instanceof Record and + not exists(TypeVariable tv | tv.getGenericType() = documentable | + "<" + tv.getName() + ">" = paramTag.getParamName() + ) and + not documentable.(Record).getCanonicalConstructor().getAParameter().getName() = + paramTag.getParamName() and + msg = + "@param tag \"" + paramTag.getParamName() + + "\" does not match any actual type parameter or record parameter of record \"" + + documentable.getName() + "\"." else // The tag has no value at all. msg = "This @param tag does not have a value." diff --git a/java/ql/src/change-notes/2024-04-02-javadoc-records.md b/java/ql/src/change-notes/2024-04-02-javadoc-records.md new file mode 100644 index 00000000000..e3859c9618b --- /dev/null +++ b/java/ql/src/change-notes/2024-04-02-javadoc-records.md @@ -0,0 +1,5 @@ +--- +category: minorAnalysis +--- +* The `java/unknown-javadoc-parameter` now accepts `@param` tags that apply to the parameters of a + record. diff --git a/java/ql/test/query-tests/SpuriousJavadocParam/Test.java b/java/ql/test/query-tests/SpuriousJavadocParam/Test.java index 84e05540c05..de108008ee8 100644 --- a/java/ql/test/query-tests/SpuriousJavadocParam/Test.java +++ b/java/ql/test/query-tests/SpuriousJavadocParam/Test.java @@ -120,5 +120,18 @@ public class Test { */ interface GenericInterface {} + /** + * @param i exists + * @param k does not + */ + static record SomeRecord(int i, int j) {} + + /** + * @param exists + * @param i exists + * @param k does not + */ + static record GenericRecord(int i, int j) {} + // Diagnostic Matches: Incomplete inheritance relation for type java.lang.Object and supertype none } diff --git a/java/ql/test/query-tests/SpuriousJavadocParam/options b/java/ql/test/query-tests/SpuriousJavadocParam/options new file mode 100644 index 00000000000..fc57fe025b9 --- /dev/null +++ b/java/ql/test/query-tests/SpuriousJavadocParam/options @@ -0,0 +1 @@ +//semmle-extractor-options: --javac-args -source 16 -target 16 diff --git a/java/ql/test/query-tests/SpuriousJavadocParam/test.expected b/java/ql/test/query-tests/SpuriousJavadocParam/test.expected index 42bdb93e60c..0ade1a7d519 100644 --- a/java/ql/test/query-tests/SpuriousJavadocParam/test.expected +++ b/java/ql/test/query-tests/SpuriousJavadocParam/test.expected @@ -12,3 +12,5 @@ | Test.java:112:6:112:12 | @param | @param tag "" does not match any actual type parameter of type "GenericClass". | | Test.java:118:6:118:12 | @param | @param tag "T" does not match any actual type parameter of type "GenericInterface". | | Test.java:119:6:119:12 | @param | @param tag "" does not match any actual type parameter of type "GenericInterface". | +| Test.java:125:6:125:12 | @param | @param tag "k" does not match any actual type parameter or record parameter of record "SomeRecord". | +| Test.java:132:6:132:12 | @param | @param tag "k" does not match any actual type parameter or record parameter of record "GenericRecord". | From 2336e1462749386aa3db178df112d81c2f569e09 Mon Sep 17 00:00:00 2001 From: Dave Bartolomeo Date: Thu, 4 Apr 2024 10:31:05 -0400 Subject: [PATCH 10/50] Remove expectation of spurious diagnostic --- java/ql/test/query-tests/SpuriousJavadocParam/Test.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/java/ql/test/query-tests/SpuriousJavadocParam/Test.java b/java/ql/test/query-tests/SpuriousJavadocParam/Test.java index de108008ee8..434353f4c67 100644 --- a/java/ql/test/query-tests/SpuriousJavadocParam/Test.java +++ b/java/ql/test/query-tests/SpuriousJavadocParam/Test.java @@ -132,6 +132,4 @@ public class Test { * @param k does not */ static record GenericRecord(int i, int j) {} - - // Diagnostic Matches: Incomplete inheritance relation for type java.lang.Object and supertype none } From 7bec41096c182364830c260ce49489d593ca9550 Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 3 Apr 2024 14:18:30 +0000 Subject: [PATCH 11/50] Python: Rename `tsg-build` target to `tsp-build` The latter makes more sense, as it's actually building `tree-sitter-python`. --- python/extractor/tsg-python/tsp/BUILD.bazel | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/extractor/tsg-python/tsp/BUILD.bazel b/python/extractor/tsg-python/tsp/BUILD.bazel index 71319e894f6..e3389fce1cc 100644 --- a/python/extractor/tsg-python/tsp/BUILD.bazel +++ b/python/extractor/tsg-python/tsp/BUILD.bazel @@ -7,7 +7,7 @@ package(default_visibility = ["//visibility:public"]) # This will run the build script from the root of the workspace, and # collect the outputs. cargo_build_script( - name = "tsg-build", + name = "tsp-build", srcs = ["bindings/rust/build.rs"], data = glob([ "src/**", @@ -32,7 +32,7 @@ rust_library( proc_macro_deps = all_crate_deps( proc_macro = True, ), - deps = [":tsg-build"] + all_crate_deps( + deps = [":tsp-build"] + all_crate_deps( normal = True, ), ) From 752d28c1b9fee6047180b7233e753473028b557a Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 3 Apr 2024 14:20:54 +0000 Subject: [PATCH 12/50] Python: Update repinning instructions This aligns us better with the corresponding instructions for the Ruby extractor. --- python/extractor/tsg-python/Cargo.toml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/python/extractor/tsg-python/Cargo.toml b/python/extractor/tsg-python/Cargo.toml index feecd254159..ac4d4093189 100644 --- a/python/extractor/tsg-python/Cargo.toml +++ b/python/extractor/tsg-python/Cargo.toml @@ -7,9 +7,14 @@ authors = ["Taus Brock-Nannestad "] edition = "2018" # When changing/updating these, the `Cargo.Bazel.lock` file has to be regenerated. -# Check out the documentation at https://bazelbuild.github.io/rules_rust/crate_universe.html#repinning--updating-dependencies -# for how to do so. The bazel repository for the tsg-python project is called `py_deps`, -# and instead of calling `bazel sync`, `./build --bazel sync` should be used instead, to always use the correct bazel version. +# Run `CARGO_BAZEL_REPIN=true CARGO_BAZEL_REPIN_ONLY=py_deps ./build --bazel sync --only=py_deps` +# in the `semmle-code` repository to do so. +# For more information, check out the documentation at +# https://bazelbuild.github.io/rules_rust/crate_universe.html#repinning--updating-dependencies +# In the future, the hope is to move this handling of the dependencies entirely into the `codeql` repository, +# but that depends on `rules_rust` being fully compatible with bzlmod, which they aren't yet +# (c.f. https://github.com/bazelbuild/rules_rust/issues/2452). +# Warning: The process takes >5min on my M1 mac, so do wait for a while. [dependencies] anyhow = "1.0" regex = "1" From 599f573a4abb0b20411aa22298dd3ec0c5a1eaf1 Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 3 Apr 2024 14:25:03 +0000 Subject: [PATCH 13/50] Python: Preserve comments and docstrings in extractor --- python/extractor/make_zips.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/python/extractor/make_zips.py b/python/extractor/make_zips.py index b91b1bf458d..faf595acc37 100755 --- a/python/extractor/make_zips.py +++ b/python/extractor/make_zips.py @@ -8,7 +8,6 @@ import optparse import compileall from python_tracer import getzipfilename -from unparse import strip_comments_and_docstrings # TO DO -- Add options to set destination directory and source directory @@ -84,9 +83,7 @@ def write_source(zipped, root, name, extensions=[".py"]): if ext not in extensions: continue path = os.path.join(dirpath, name) - temp = strip_comments_and_docstrings(path) - zipped.write(temp, os.path.relpath(path, root)) - os.remove(temp) + zipped.write(path, os.path.relpath(path, root)) def main(): parser = optparse.OptionParser(usage = "usage: %prog [install-dir]") From ef9f99b3be728d910539c7750e77a11cfda64425 Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 3 Apr 2024 14:30:29 +0000 Subject: [PATCH 14/50] Python: Remove `unparse.py` --- python/extractor/BUILD.bazel | 1 - python/extractor/licenses.md | 1 - python/extractor/unparse.py | 709 ----------------------------------- 3 files changed, 711 deletions(-) delete mode 100644 python/extractor/unparse.py diff --git a/python/extractor/BUILD.bazel b/python/extractor/BUILD.bazel index 3b5a5b3a617..697bf8d49a4 100644 --- a/python/extractor/BUILD.bazel +++ b/python/extractor/BUILD.bazel @@ -5,7 +5,6 @@ py_binary( srcs = [ "make_zips.py", "python_tracer.py", - "unparse.py", ], data = [ "LICENSE-PSF.md", diff --git a/python/extractor/licenses.md b/python/extractor/licenses.md index f3a5c6cc458..d585a84626d 100644 --- a/python/extractor/licenses.md +++ b/python/extractor/licenses.md @@ -6,7 +6,6 @@ | `tsg-python/tree-sitter-python` | Y | MIT | Used in `tsg-python` to parse Python files | | `tsg-python` | Y | MIT / Apache | This is our own creation, so are free to choose what license it is covered by. | | `tree-sitter-graph` | N | MIT / Apache | Used in `tsg-python` to execute files written in the `tree-sitter-graph` language. | -| `unparse.py` | Y | PSF | Copied and adapted from `Tools/unparse.py` from the `cpython` source code, with attribution. | | `imp.py` | Y | PSF | Copied and adapted from `Lib/imp.py` from the `cpython` source code, with attribution. | | `semmle/data/*.trap` | Y | PSF | These files were derived from the C source code of the `cpython` project, and are used in our modelling of built-in objects. No attribution, currently. | | `semmle/thrift/parse.py` | Y | Apache | Includes a grammar based on https://github.com/apache/thrift/blob/master/doc/specs/idl.md, with comment stating this attribution. | diff --git a/python/extractor/unparse.py b/python/extractor/unparse.py deleted file mode 100644 index b12f1501592..00000000000 --- a/python/extractor/unparse.py +++ /dev/null @@ -1,709 +0,0 @@ -#Copied Tools.unparse.py with modifications. Copyright PSF. - -"Usage: unparse.py " -import sys -import ast -import tokenize -import io -import os -import shutil - -# Large float and imaginary literals get turned into infinities in the AST. -# We unparse those infinities to INFSTR. -INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) - -def interleave(inter, f, seq): - """Call f on each item in seq, calling inter() in between. - """ - seq = iter(seq) - try: - f(next(seq)) - except StopIteration: - pass - else: - for x in seq: - inter() - f(x) - -class Unparser: - """Methods in this class recursively traverse an AST and - output source code for the abstract syntax; original formatting - is disregarded. """ - - def __init__(self, tree, file = sys.stdout): - """Unparser(tree, file=sys.stdout) -> None. - Print the source for tree to file.""" - self.f = file - self._indent = 0 - self.dispatch(tree) - print("", file=self.f) - self.f.flush() - - def fill(self, text = ""): - "Indent a piece of text, according to the current indentation level" - self.f.write("\n"+" "*self._indent + text) - - def write(self, text): - "Append a piece of text to the current line." - self.f.write(text) - - def enter(self): - "Print ':', and increase the indentation." - self.write(":") - self._indent += 1 - - def leave(self): - "Decrease the indentation level." - self._indent -= 1 - - def dispatch(self, tree): - "Dispatcher function, dispatching tree type T to method _T." - if isinstance(tree, list): - for t in tree: - self.dispatch(t) - return - meth = getattr(self, "_"+tree.__class__.__name__) - meth(tree) - - def remove_docstring(self, t): - if hasattr(t, "docstring"): - return - if not t.body: - return - if not isinstance(t.body[0], ast.Expr): - return - if not isinstance(t.body[0].value, ast.Str): - return - t.body = t.body[1:] - - def add_pass(self, t): - if t.body: - #No pass needed - return - t.body = [ast.Pass()] - - ############### Unparsing methods ###################### - # There should be one method per concrete grammar type # - # Constructors should be grouped by sum type. Ideally, # - # this would follow the order in the grammar, but # - # currently doesn't. # - ######################################################## - - def _Module(self, tree): - self.remove_docstring(tree) - self.add_pass(tree) - for stmt in tree.body: - self.dispatch(stmt) - - # stmt - def _Expr(self, tree): - self.fill() - self.dispatch(tree.value) - - def _Import(self, t): - self.fill("import ") - interleave(lambda: self.write(", "), self.dispatch, t.names) - - def _ImportFrom(self, t): - self.fill("from ") - self.write("." * t.level) - if t.module: - self.write(t.module) - self.write(" import ") - interleave(lambda: self.write(", "), self.dispatch, t.names) - - def _Assign(self, t): - self.fill() - for target in t.targets: - self.dispatch(target) - self.write(" = ") - self.dispatch(t.value) - - def _AugAssign(self, t): - self.fill() - self.dispatch(t.target) - self.write(" "+self.binop[t.op.__class__.__name__]+"= ") - self.dispatch(t.value) - - def _AnnAssign(self, t): - self.fill() - if not t.simple and isinstance(t.target, ast.Name): - self.write('(') - self.dispatch(t.target) - if not t.simple and isinstance(t.target, ast.Name): - self.write(')') - self.write(": ") - self.dispatch(t.annotation) - if t.value: - self.write(" = ") - self.dispatch(t.value) - - def _Return(self, t): - self.fill("return") - if t.value: - self.write(" ") - self.dispatch(t.value) - - def _Pass(self, t): - self.fill("pass") - - def _Break(self, t): - self.fill("break") - - def _Continue(self, t): - self.fill("continue") - - def _Delete(self, t): - self.fill("del ") - interleave(lambda: self.write(", "), self.dispatch, t.targets) - - def _Assert(self, t): - self.fill("assert ") - self.dispatch(t.test) - if t.msg: - self.write(", ") - self.dispatch(t.msg) - - def _Global(self, t): - self.fill("global ") - interleave(lambda: self.write(", "), self.write, t.names) - - def _Nonlocal(self, t): - self.fill("nonlocal ") - interleave(lambda: self.write(", "), self.write, t.names) - - def _Await(self, t): - self.write("(") - self.write("await") - if t.value: - self.write(" ") - self.dispatch(t.value) - self.write(")") - - def _Yield(self, t): - self.write("(") - self.write("yield") - if t.value: - self.write(" ") - self.dispatch(t.value) - self.write(")") - - def _YieldFrom(self, t): - self.write("(") - self.write("yield from") - if t.value: - self.write(" ") - self.dispatch(t.value) - self.write(")") - - def _Raise(self, t): - self.fill("raise") - if not t.exc: - assert not t.cause - return - self.write(" ") - self.dispatch(t.exc) - if t.cause: - self.write(" from ") - self.dispatch(t.cause) - - def _Try(self, t): - self.fill("try") - self.enter() - self.dispatch(t.body) - self.leave() - for ex in t.handlers: - self.dispatch(ex) - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - if t.finalbody: - self.fill("finally") - self.enter() - self.dispatch(t.finalbody) - self.leave() - - def _ExceptHandler(self, t): - self.fill("except") - if t.type: - self.write(" ") - self.dispatch(t.type) - if t.name: - self.write(" as ") - self.write(t.name) - self.enter() - self.dispatch(t.body) - self.leave() - - def _ClassDef(self, t): - self.write("\n") - for deco in t.decorator_list: - self.fill("@") - self.dispatch(deco) - self.fill("class "+t.name) - self.write("(") - comma = False - for e in t.bases: - if comma: self.write(", ") - else: comma = True - self.dispatch(e) - for e in t.keywords: - if comma: self.write(", ") - else: comma = True - self.dispatch(e) - self.write(")") - - self.enter() - self.remove_docstring(t) - self.add_pass(t) - self.dispatch(t.body) - self.leave() - - def _FunctionDef(self, t): - self.__FunctionDef_helper(t, "def") - - def _AsyncFunctionDef(self, t): - self.__FunctionDef_helper(t, "async def") - - def __FunctionDef_helper(self, t, fill_suffix): - self.write("\n") - for deco in t.decorator_list: - self.fill("@") - self.dispatch(deco) - def_str = fill_suffix+" "+t.name + "(" - self.fill(def_str) - self.dispatch(t.args) - self.write(")") - if t.returns: - self.write(" -> ") - self.dispatch(t.returns) - self.enter() - self.remove_docstring(t) - self.add_pass(t) - self.dispatch(t.body) - self.leave() - - def _For(self, t): - self.__For_helper("for ", t) - - def _AsyncFor(self, t): - self.__For_helper("async for ", t) - - def __For_helper(self, fill, t): - self.fill(fill) - self.dispatch(t.target) - self.write(" in ") - self.dispatch(t.iter) - self.enter() - self.dispatch(t.body) - self.leave() - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - - def _If(self, t): - self.fill("if ") - self.dispatch(t.test) - self.enter() - self.dispatch(t.body) - self.leave() - # collapse nested ifs into equivalent elifs. - while (t.orelse and len(t.orelse) == 1 and - isinstance(t.orelse[0], ast.If)): - t = t.orelse[0] - self.fill("elif ") - self.dispatch(t.test) - self.enter() - self.dispatch(t.body) - self.leave() - # final else - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - - def _While(self, t): - self.fill("while ") - self.dispatch(t.test) - self.enter() - self.dispatch(t.body) - self.leave() - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - - def _With(self, t): - self.fill("with ") - interleave(lambda: self.write(", "), self.dispatch, t.items) - self.enter() - self.dispatch(t.body) - self.leave() - - def _AsyncWith(self, t): - self.fill("async with ") - interleave(lambda: self.write(", "), self.dispatch, t.items) - self.enter() - self.dispatch(t.body) - self.leave() - - # expr - def _Bytes(self, t): - self.write(repr(t.s)) - - def _Str(self, tree): - s = repr(tree.s).encode("ascii", errors="backslashreplace").decode("ascii") - self.write(s) - - def _JoinedStr(self, t): - self.write("f") - string = io.StringIO() - self._fstring_JoinedStr(t, string.write) - self.write(repr(string.getvalue())) - - def _FormattedValue(self, t): - self.write("f") - string = io.StringIO() - self._fstring_FormattedValue(t, string.write) - self.write(repr(string.getvalue())) - - def _fstring_JoinedStr(self, t, write): - for value in t.values: - meth = getattr(self, "_fstring_" + type(value).__name__) - meth(value, write) - - def _fstring_Str(self, t, write): - value = t.s.replace("{", "{{").replace("}", "}}") - write(value) - - def _fstring_Constant(self, t, write): - assert isinstance(t.value, str) - value = t.value.replace("{", "{{").replace("}", "}}") - write(value) - - def _fstring_FormattedValue(self, t, write): - write("{") - expr = io.StringIO() - Unparser(t.value, expr) - expr = expr.getvalue().rstrip("\n") - if expr.startswith("{"): - write(" ") # Separate pair of opening brackets as "{ {" - write(expr) - if t.conversion != -1: - conversion = chr(t.conversion) - assert conversion in "sra" - write("!%s" % conversion) - if t.format_spec: - write(":") - meth = getattr(self, "_fstring_" + type(t.format_spec).__name__) - meth(t.format_spec, write) - write("}") - - def _Name(self, t): - self.write(t.id) - - def _write_constant(self, value): - if isinstance(value, (float, complex)): - self.write(repr(value).replace("inf", INFSTR)) - else: - self.write(repr(value)) - - def _Constant(self, t): - value = t.value - if isinstance(value, tuple): - self.write("(") - if len(value) == 1: - self._write_constant(value[0]) - self.write(",") - else: - interleave(lambda: self.write(", "), self._write_constant, value) - self.write(")") - else: - self._write_constant(t.value) - - def _NameConstant(self, t): - self.write(repr(t.value)) - - def _Num(self, t): - # Substitute overflowing decimal literal for AST infinities. - self.write(repr(t.n).replace("inf", INFSTR)) - - def _List(self, t): - self.write("[") - interleave(lambda: self.write(", "), self.dispatch, t.elts) - self.write("]") - - def _ListComp(self, t): - self.write("[") - self.dispatch(t.elt) - for gen in t.generators: - self.dispatch(gen) - self.write("]") - - def _GeneratorExp(self, t): - self.write("(") - self.dispatch(t.elt) - for gen in t.generators: - self.dispatch(gen) - self.write(")") - - def _SetComp(self, t): - self.write("{") - self.dispatch(t.elt) - for gen in t.generators: - self.dispatch(gen) - self.write("}") - - def _DictComp(self, t): - self.write("{") - self.dispatch(t.key) - self.write(": ") - self.dispatch(t.value) - for gen in t.generators: - self.dispatch(gen) - self.write("}") - - def _comprehension(self, t): - if hasattr(t, "is_async") and t.is_async: - self.write(" async for ") - else: - self.write(" for ") - self.dispatch(t.target) - self.write(" in ") - self.dispatch(t.iter) - for if_clause in t.ifs: - self.write(" if ") - self.dispatch(if_clause) - - def _IfExp(self, t): - self.write("(") - self.dispatch(t.body) - self.write(" if ") - self.dispatch(t.test) - self.write(" else ") - self.dispatch(t.orelse) - self.write(")") - - def _Set(self, t): - assert(t.elts) # should be at least one element - self.write("{") - interleave(lambda: self.write(", "), self.dispatch, t.elts) - self.write("}") - - def _Dict(self, t): - self.write("{") - def write_key_value_pair(k, v): - self.dispatch(k) - self.write(": ") - self.dispatch(v) - - def write_item(item): - k, v = item - if k is None: - # for dictionary unpacking operator in dicts {**{'y': 2}} - # see PEP 448 for details - self.write("**") - self.dispatch(v) - else: - write_key_value_pair(k, v) - interleave(lambda: self.write(", "), write_item, zip(t.keys, t.values)) - self.write("}") - - def _Tuple(self, t): - self.write("(") - if len(t.elts) == 1: - elt = t.elts[0] - self.dispatch(elt) - self.write(",") - else: - interleave(lambda: self.write(", "), self.dispatch, t.elts) - self.write(")") - - unop = {"Invert":"~", "Not": "not", "UAdd":"+", "USub":"-"} - def _UnaryOp(self, t): - self.write("(") - self.write(self.unop[t.op.__class__.__name__]) - self.write(" ") - self.dispatch(t.operand) - self.write(")") - - binop = { "Add":"+", "Sub":"-", "Mult":"*", "MatMult":"@", "Div":"/", "Mod":"%", - "LShift":"<<", "RShift":">>", "BitOr":"|", "BitXor":"^", "BitAnd":"&", - "FloorDiv":"//", "Pow": "**"} - def _BinOp(self, t): - self.write("(") - self.dispatch(t.left) - self.write(" " + self.binop[t.op.__class__.__name__] + " ") - self.dispatch(t.right) - self.write(")") - - cmpops = {"Eq":"==", "NotEq":"!=", "Lt":"<", "LtE":"<=", "Gt":">", "GtE":">=", - "Is":"is", "IsNot":"is not", "In":"in", "NotIn":"not in"} - def _Compare(self, t): - self.write("(") - self.dispatch(t.left) - for o, e in zip(t.ops, t.comparators): - self.write(" " + self.cmpops[o.__class__.__name__] + " ") - self.dispatch(e) - self.write(")") - - boolops = {ast.And: 'and', ast.Or: 'or'} - def _BoolOp(self, t): - self.write("(") - s = " %s " % self.boolops[t.op.__class__] - interleave(lambda: self.write(s), self.dispatch, t.values) - self.write(")") - - def _Attribute(self,t): - self.dispatch(t.value) - # Special case: 3.__abs__() is a syntax error, so if t.value - # is an integer literal then we need to either parenthesize - # it or add an extra space to get 3 .__abs__(). - if isinstance(t.value, ast.Num) and isinstance(t.value.n, int): - self.write(" ") - self.write(".") - self.write(t.attr) - - def _Call(self, t): - self.dispatch(t.func) - self.write("(") - comma = False - for e in t.args: - if comma: self.write(", ") - else: comma = True - self.dispatch(e) - for e in t.keywords: - if comma: self.write(", ") - else: comma = True - self.dispatch(e) - self.write(")") - - def _Subscript(self, t): - self.dispatch(t.value) - self.write("[") - self.dispatch(t.slice) - self.write("]") - - def _Starred(self, t): - self.write("*") - self.dispatch(t.value) - - # slice - def _Ellipsis(self, t): - self.write("...") - - def _Index(self, t): - self.dispatch(t.value) - - def _Slice(self, t): - if t.lower: - self.dispatch(t.lower) - self.write(":") - if t.upper: - self.dispatch(t.upper) - if t.step: - self.write(":") - self.dispatch(t.step) - - def _ExtSlice(self, t): - interleave(lambda: self.write(', '), self.dispatch, t.dims) - - # argument - def _arg(self, t): - self.write(t.arg) - if t.annotation: - self.write(": ") - self.dispatch(t.annotation) - - # others - def _arguments(self, t): - first = True - # normal arguments - defaults = [None] * (len(t.args) - len(t.defaults)) + t.defaults - for a, d in zip(t.args, defaults): - if first:first = False - else: self.write(", ") - self.dispatch(a) - if d: - self.write("=") - self.dispatch(d) - - # varargs, or bare '*' if no varargs but keyword-only arguments present - if t.vararg or t.kwonlyargs: - if first:first = False - else: self.write(", ") - self.write("*") - if t.vararg: - self.write(t.vararg.arg) - if t.vararg.annotation: - self.write(": ") - self.dispatch(t.vararg.annotation) - - # keyword-only arguments - if t.kwonlyargs: - for a, d in zip(t.kwonlyargs, t.kw_defaults): - if first:first = False - else: self.write(", ") - self.dispatch(a), - if d: - self.write("=") - self.dispatch(d) - - # kwargs - if t.kwarg: - if first:first = False - else: self.write(", ") - self.write("**"+t.kwarg.arg) - if t.kwarg.annotation: - self.write(": ") - self.dispatch(t.kwarg.annotation) - - def _keyword(self, t): - if t.arg is None: - self.write("**") - else: - self.write(t.arg) - self.write("=") - self.dispatch(t.value) - - def _Lambda(self, t): - self.write("(") - self.write("lambda ") - self.dispatch(t.args) - self.write(": ") - self.dispatch(t.body) - self.write(")") - - def _alias(self, t): - self.write(t.name) - if t.asname: - self.write(" as "+t.asname) - - def _withitem(self, t): - self.dispatch(t.context_expr) - if t.optional_vars: - self.write(" as ") - self.dispatch(t.optional_vars) - -def roundtrip(filename, outpath): - with open(filename, "rb") as pyfile: - encoding = tokenize.detect_encoding(pyfile.readline)[0] - with open(filename, "r", encoding=encoding) as pyfile: - source = pyfile.read() - tree = compile(source, filename, "exec", ast.PyCF_ONLY_AST) - with open(outpath, "w", encoding=encoding) as output: - Unparser(tree, output) - -def strip_comments_and_docstrings(path): - tmp = path + ".tmp" - if path.endswith(".py"): - roundtrip(path, tmp) - else: - shutil.copy(path, tmp) - return tmp From 2fb9c2db6fc53c8adf1ce46b3d084bc4a6565b0c Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Tue, 12 Mar 2024 15:47:15 +0100 Subject: [PATCH 15/50] C#: Remove deprecated qualifiedName predicates. --- csharp/ql/lib/semmle/code/csharp/Element.qll | 29 -------------- csharp/ql/lib/semmle/code/csharp/Member.qll | 38 ------------------- .../ql/lib/semmle/code/csharp/Namespace.qll | 10 ----- 3 files changed, 77 deletions(-) diff --git a/csharp/ql/lib/semmle/code/csharp/Element.qll b/csharp/ql/lib/semmle/code/csharp/Element.qll index 2c69912c993..ccfd1a27925 100644 --- a/csharp/ql/lib/semmle/code/csharp/Element.qll +++ b/csharp/ql/lib/semmle/code/csharp/Element.qll @@ -101,25 +101,6 @@ class NamedElement extends Element, @named_element { /** Holds if this element has name 'name'. */ final predicate hasName(string name) { name = this.getName() } - /** - * Gets the fully qualified name of this element, for example the - * fully qualified name of `M` on line 3 is `N.C.M` in - * - * ```csharp - * namespace N { - * class C { - * void M(int i, string s) { } - * } - * } - * ``` - */ - cached - deprecated final string getQualifiedName() { - exists(string qualifier, string name | this.hasQualifiedName(qualifier, name) | - if qualifier = "" then result = name else result = qualifier + "." + name - ) - } - /** * Gets the fully qualified name of this element, for example the * fully qualified name of `M` on line 3 is `N.C.M` in @@ -142,16 +123,6 @@ class NamedElement extends Element, @named_element { ) } - /** - * DEPRECATED: Use `hasFullyQualifiedName` instead. - * - * Holds if this element has the qualified name `qualifier`.`name`. - */ - cached - deprecated predicate hasQualifiedName(string qualifier, string name) { - qualifier = "" and name = this.getName() - } - /** Holds if this element has the fully qualified name `qualifier`.`name`. */ cached predicate hasFullyQualifiedName(string qualifier, string name) { diff --git a/csharp/ql/lib/semmle/code/csharp/Member.qll b/csharp/ql/lib/semmle/code/csharp/Member.qll index 1be091170e3..a2758f03942 100644 --- a/csharp/ql/lib/semmle/code/csharp/Member.qll +++ b/csharp/ql/lib/semmle/code/csharp/Member.qll @@ -71,37 +71,10 @@ class Declaration extends NamedElement, @declaration { override string toString() { result = this.getName() } - deprecated override predicate hasQualifiedName(string qualifier, string name) { - QualifiedName::hasQualifiedName(this, qualifier, name) - } - override predicate hasFullyQualifiedName(string qualifier, string name) { QualifiedName::hasQualifiedName(this, qualifier, name) } - /** - * DEPRECATED: Use `getFullyQualifiedNameWithTypes` instead. - * - * Gets the fully qualified name of this declaration, including types, for example - * the fully qualified name with types of `M` on line 3 is `N.C.M(int, string)` in - * - * ```csharp - * namespace N { - * class C { - * void M(int i, string s) { } - * } - * } - * ``` - */ - deprecated string getQualifiedNameWithTypes() { - exists(string qual | - qual = this.getDeclaringType().getQualifiedName() and - if this instanceof NestedType - then result = qual + "+" + this.toStringWithTypes() - else result = qual + "." + this.toStringWithTypes() - ) - } - /** * Gets the fully qualified name of this declaration, including types, for example * the fully qualified name with types of `M` on line 3 is `N.C.M(int, string)` in @@ -263,17 +236,6 @@ class Member extends Modifiable, @member { /** Gets an access to this member. */ MemberAccess getAnAccess() { result.getTarget() = this } - /** - * DEPRECATED: Use `hasFullyQualifiedName` instead. - * - * Holds if this member has name `name` and is defined in type `type` - * with namespace `namespace`. - */ - cached - deprecated final predicate hasQualifiedName(string namespace, string type, string name) { - QualifiedName::hasQualifiedName(this, namespace, type, name) - } - /** * Holds if this member has name `name` and is defined in type `type` * with namespace `namespace`. diff --git a/csharp/ql/lib/semmle/code/csharp/Namespace.qll b/csharp/ql/lib/semmle/code/csharp/Namespace.qll index 002ce444b3f..383fe8ab2a8 100644 --- a/csharp/ql/lib/semmle/code/csharp/Namespace.qll +++ b/csharp/ql/lib/semmle/code/csharp/Namespace.qll @@ -38,16 +38,6 @@ class Namespace extends TypeContainer, Declaration, @namespace { parent_namespace(result, this) } - /** - * Holds if this namespace has the qualified name `qualifier`.`name`. - * - * For example if the qualified name is `System.Collections.Generic`, then - * `qualifier`=`System.Collections` and `name`=`Generic`. - */ - deprecated override predicate hasQualifiedName(string qualifier, string name) { - namespaceHasQualifiedName(this, qualifier, name) - } - /** * Holds if this namespace has the qualified name `qualifier`.`name`. * From 8fa91914349bf21e52fabfde00c68f13b28c8a63 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Wed, 13 Mar 2024 10:12:22 +0100 Subject: [PATCH 16/50] C#: Deprecate the getFullyQualifiedName predicate. --- .../diag_recursive_generics/Types.ql | 7 ++++--- csharp/ql/lib/semmle/code/csharp/Element.qll | 4 +++- csharp/ql/lib/semmle/code/csharp/Member.qll | 9 +++++---- csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll | 8 +++++++- .../csharp/dataflow/internal/FlowSummaryImpl.qll | 15 ++++++++++++--- .../security/dataflow/ExternalAPIsQuery.qll | 6 +++--- .../library-tests/constructors/Destructors1.ql | 2 +- 7 files changed, 35 insertions(+), 16 deletions(-) diff --git a/csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql b/csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql index d59c60ec802..fff011dcbd5 100644 --- a/csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql +++ b/csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql @@ -1,5 +1,6 @@ import csharp +import semmle.code.csharp.commons.QualifiedName -from Class c -where c.fromSource() -select c, c.getBaseClass().getFullyQualifiedName() +from Class c, string qualifier, string name +where c.fromSource() and c.getBaseClass().hasFullyQualifiedName(qualifier, name) +select c, getQualifiedName(qualifier, name) diff --git a/csharp/ql/lib/semmle/code/csharp/Element.qll b/csharp/ql/lib/semmle/code/csharp/Element.qll index ccfd1a27925..38176acfd92 100644 --- a/csharp/ql/lib/semmle/code/csharp/Element.qll +++ b/csharp/ql/lib/semmle/code/csharp/Element.qll @@ -102,6 +102,8 @@ class NamedElement extends Element, @named_element { final predicate hasName(string name) { name = this.getName() } /** + * DEPRECATED: Use `hasFullyQualifiedName` instead. + * * Gets the fully qualified name of this element, for example the * fully qualified name of `M` on line 3 is `N.C.M` in * @@ -117,7 +119,7 @@ class NamedElement extends Element, @named_element { * ``System.Collections.Generic.IList`1``. */ cached - final string getFullyQualifiedName() { + deprecated final string getFullyQualifiedName() { exists(string qualifier, string name | this.hasFullyQualifiedName(qualifier, name) | if qualifier = "" then result = name else result = qualifier + "." + name ) diff --git a/csharp/ql/lib/semmle/code/csharp/Member.qll b/csharp/ql/lib/semmle/code/csharp/Member.qll index a2758f03942..99bf8eaa4f0 100644 --- a/csharp/ql/lib/semmle/code/csharp/Member.qll +++ b/csharp/ql/lib/semmle/code/csharp/Member.qll @@ -88,11 +88,12 @@ class Declaration extends NamedElement, @declaration { * ``` */ string getFullyQualifiedNameWithTypes() { - exists(string qual | - qual = this.getDeclaringType().getFullyQualifiedName() and + exists(string fullqual, string qual, string name | + this.getDeclaringType().hasFullyQualifiedName(qual, name) and + fullqual = getQualifiedName(qual, name) and if this instanceof NestedType - then result = qual + "+" + this.toStringWithTypes() - else result = qual + "." + this.toStringWithTypes() + then result = fullqual + "+" + this.toStringWithTypes() + else result = fullqual + "." + this.toStringWithTypes() ) } diff --git a/csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll b/csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll index 673bd1a5638..9cd33ea260f 100644 --- a/csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll +++ b/csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll @@ -3,6 +3,7 @@ */ import csharp +private import semmle.code.csharp.commons.QualifiedName /** * Provides classes for working with static single assignment (SSA) form. @@ -120,7 +121,12 @@ module Ssa { result = prefix + "." + this.getAssignable() | if f.(Modifiable).isStatic() - then prefix = f.getDeclaringType().getFullyQualifiedName() + then + exists(string qualifier, string name | + f.getDeclaringType().hasFullyQualifiedName(qualifier, name) + | + prefix = getQualifiedName(qualifier, name) + ) else prefix = "this" ) } diff --git a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll index 96f45e77655..82c2a7dcd3d 100644 --- a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll +++ b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll @@ -3,6 +3,7 @@ */ private import csharp +private import semmle.code.csharp.commons.QualifiedName private import semmle.code.csharp.frameworks.system.linq.Expressions private import codeql.dataflow.internal.FlowSummaryImpl private import codeql.dataflow.internal.AccessPathSyntax as AccessPath @@ -42,10 +43,18 @@ module Input implements InputSig string encodeContent(ContentSet c, string arg) { c = TElementContent() and result = "Element" and arg = "" or - exists(Field f | c = TFieldContent(f) and result = "Field" and arg = f.getFullyQualifiedName()) + exists(Field f, string qualifier, string name | + c = TFieldContent(f) and + f.hasFullyQualifiedName(qualifier, name) and + arg = getQualifiedName(qualifier, name) and + result = "Field" + ) or - exists(Property p | - c = TPropertyContent(p) and result = "Property" and arg = p.getFullyQualifiedName() + exists(Property p, string qualifier, string name | + c = TPropertyContent(p) and + p.hasFullyQualifiedName(qualifier, name) and + arg = getQualifiedName(qualifier, name) and + result = "Property" ) or exists(SyntheticField f | diff --git a/csharp/ql/lib/semmle/code/csharp/security/dataflow/ExternalAPIsQuery.qll b/csharp/ql/lib/semmle/code/csharp/security/dataflow/ExternalAPIsQuery.qll index 3075fe53a87..a0d0ada957a 100644 --- a/csharp/ql/lib/semmle/code/csharp/security/dataflow/ExternalAPIsQuery.qll +++ b/csharp/ql/lib/semmle/code/csharp/security/dataflow/ExternalAPIsQuery.qll @@ -139,13 +139,13 @@ class ExternalApiUsedWithUntrustedData extends TExternalApi { /** Gets a textual representation of this element. */ string toString() { - exists(Callable m, int index, string indexString | + exists(Callable m, int index, string indexString, string qualifier, string name | if index = -1 then indexString = "qualifier" else indexString = "param " + index | this = TExternalApiParameter(m, index) and + m.getDeclaringType().hasFullyQualifiedName(qualifier, name) and result = - m.getDeclaringType().getFullyQualifiedName() + "." + m.toStringWithTypes() + " [" + - indexString + "]" + getQualifiedName(qualifier, name) + "." + m.toStringWithTypes() + " [" + indexString + "]" ) } } diff --git a/csharp/ql/test/library-tests/constructors/Destructors1.ql b/csharp/ql/test/library-tests/constructors/Destructors1.ql index 368980a290c..673a0e941f0 100644 --- a/csharp/ql/test/library-tests/constructors/Destructors1.ql +++ b/csharp/ql/test/library-tests/constructors/Destructors1.ql @@ -10,4 +10,4 @@ where c.getDeclaringType().hasFullyQualifiedName(qualifier, name) and qualifier = "Constructors" and name = "Class" -select c, c.getDeclaringType().getFullyQualifiedName() +select c, getQualifiedName(qualifier, name) From b677e89f35fd0f0ae322f0191e342520ab0bc795 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Wed, 13 Mar 2024 10:23:24 +0100 Subject: [PATCH 17/50] C#: Deprecate getFullyQualifiedNameWithTypes. --- csharp/ql/lib/semmle/code/csharp/Member.qll | 2 +- .../code/csharp/commons/QualifiedName.qll | 25 +++++++++++++++++++ .../dispatch/GetADynamicTarget.ql | 3 ++- .../frameworks/system/Dispose/Dispose.ql | 3 ++- .../frameworks/system/Equals/Equals.ql | 3 ++- .../test/library-tests/generics/Generics.ql | 12 ++++----- .../library-tests/overrides/Overrides22.ql | 3 ++- .../library-tests/unification/Unification.ql | 5 ++-- 8 files changed, 43 insertions(+), 13 deletions(-) diff --git a/csharp/ql/lib/semmle/code/csharp/Member.qll b/csharp/ql/lib/semmle/code/csharp/Member.qll index 99bf8eaa4f0..3427d4ea089 100644 --- a/csharp/ql/lib/semmle/code/csharp/Member.qll +++ b/csharp/ql/lib/semmle/code/csharp/Member.qll @@ -87,7 +87,7 @@ class Declaration extends NamedElement, @declaration { * } * ``` */ - string getFullyQualifiedNameWithTypes() { + deprecated string getFullyQualifiedNameWithTypes() { exists(string fullqual, string qual, string name | this.getDeclaringType().hasFullyQualifiedName(qual, name) and fullqual = getQualifiedName(qual, name) and diff --git a/csharp/ql/lib/semmle/code/csharp/commons/QualifiedName.qll b/csharp/ql/lib/semmle/code/csharp/commons/QualifiedName.qll index eba0fb10c7c..417dc137027 100644 --- a/csharp/ql/lib/semmle/code/csharp/commons/QualifiedName.qll +++ b/csharp/ql/lib/semmle/code/csharp/commons/QualifiedName.qll @@ -219,3 +219,28 @@ predicate splitQualifiedName(string qualifiedName, string qualifier, string name name = qualifiedName ) } + +/** + * INTERNAL: Do not use. + * + * Gets the fully qualified name of this declaration, including types, for example + * the fully qualified name with types of `M` on line 3 is `N.C.M(int, string)` in + * + * ```csharp + * namespace N { + * class C { + * void M(int i, string s) { } + * } + * } + * ``` + */ +bindingset[d] +string getFullyQualifiedNameWithTypes(Declaration d) { + exists(string fullqual, string qual, string name | + d.getDeclaringType().hasFullyQualifiedName(qual, name) and + fullqual = getQualifiedName(qual, name) and + if d instanceof NestedType + then result = fullqual + "+" + d.toStringWithTypes() + else result = fullqual + "." + d.toStringWithTypes() + ) +} diff --git a/csharp/ql/test/library-tests/dispatch/GetADynamicTarget.ql b/csharp/ql/test/library-tests/dispatch/GetADynamicTarget.ql index cbde2d43ab8..b4c94a0b507 100644 --- a/csharp/ql/test/library-tests/dispatch/GetADynamicTarget.ql +++ b/csharp/ql/test/library-tests/dispatch/GetADynamicTarget.ql @@ -1,8 +1,9 @@ import csharp +import semmle.code.csharp.commons.QualifiedName import semmle.code.csharp.dispatch.Dispatch from DispatchCall call, Callable callable where callable = call.getADynamicTarget() and callable.fromSource() -select call, callable.getFullyQualifiedNameWithTypes() +select call, getFullyQualifiedNameWithTypes(callable) diff --git a/csharp/ql/test/library-tests/frameworks/system/Dispose/Dispose.ql b/csharp/ql/test/library-tests/frameworks/system/Dispose/Dispose.ql index 943245fa0a2..5fa2f337fcf 100644 --- a/csharp/ql/test/library-tests/frameworks/system/Dispose/Dispose.ql +++ b/csharp/ql/test/library-tests/frameworks/system/Dispose/Dispose.ql @@ -1,4 +1,5 @@ import csharp +import semmle.code.csharp.commons.QualifiedName import semmle.code.csharp.frameworks.System from ValueOrRefType t, Method m, boolean b @@ -6,4 +7,4 @@ where t.fromSource() and m = getInvokedDisposeMethod(t) and if implementsDispose(t) then b = true else b = false -select t, m.getFullyQualifiedNameWithTypes(), b +select t, getFullyQualifiedNameWithTypes(m), b diff --git a/csharp/ql/test/library-tests/frameworks/system/Equals/Equals.ql b/csharp/ql/test/library-tests/frameworks/system/Equals/Equals.ql index 91c04791ef3..e6dc4ae7549 100644 --- a/csharp/ql/test/library-tests/frameworks/system/Equals/Equals.ql +++ b/csharp/ql/test/library-tests/frameworks/system/Equals/Equals.ql @@ -1,4 +1,5 @@ import csharp +import semmle.code.csharp.commons.QualifiedName import semmle.code.csharp.frameworks.System from ValueOrRefType t, Method m, boolean b @@ -6,4 +7,4 @@ where t.fromSource() and m = getInvokedEqualsMethod(t) and if implementsEquals(t) then b = true else b = false -select t, m.getFullyQualifiedNameWithTypes(), b +select t, getFullyQualifiedNameWithTypes(m), b diff --git a/csharp/ql/test/library-tests/generics/Generics.ql b/csharp/ql/test/library-tests/generics/Generics.ql index 2f3aff0fd58..71e21a5815e 100644 --- a/csharp/ql/test/library-tests/generics/Generics.ql +++ b/csharp/ql/test/library-tests/generics/Generics.ql @@ -231,18 +231,18 @@ query predicate test27(ConstructedType ct, UnboundGenericType ugt, UnboundGeneri query predicate test28(UnboundGeneric ug, string s) { ug.fromSource() and - s = ug.getFullyQualifiedNameWithTypes() + s = getFullyQualifiedNameWithTypes(ug) } query predicate test29(ConstructedGeneric cg, string s) { cg.fromSource() and - s = cg.getFullyQualifiedNameWithTypes() + s = getFullyQualifiedNameWithTypes(cg) } query predicate test30(Declaration d, string s) { d.fromSource() and d instanceof @generic and - s = d.getFullyQualifiedNameWithTypes() and + s = getFullyQualifiedNameWithTypes(d) and d != d.getUnboundDeclaration() and not d instanceof Generic } @@ -263,7 +263,7 @@ query predicate test33(ConstructedMethod cm, string s1, string s2) { exists(string namespace, string type, string name | cm.hasFullyQualifiedName(namespace, type, name) and s1 = getQualifiedName(namespace, type, name) ) and - cm.getFullyQualifiedNameWithTypes() = s2 + getFullyQualifiedNameWithTypes(cm) = s2 } query predicate test34(UnboundGeneric ug, string s1, string s2) { @@ -271,7 +271,7 @@ query predicate test34(UnboundGeneric ug, string s1, string s2) { exists(string qualifier, string name | ug.hasFullyQualifiedName(qualifier, name) and s1 = getQualifiedName(qualifier, name) ) and - ug.getFullyQualifiedNameWithTypes() = s2 + getFullyQualifiedNameWithTypes(ug) = s2 } query predicate test35(UnboundGenericMethod gm, string s1, string s2) { @@ -279,5 +279,5 @@ query predicate test35(UnboundGenericMethod gm, string s1, string s2) { exists(string namespace, string type, string name | gm.hasFullyQualifiedName(namespace, type, name) and s1 = getQualifiedName(namespace, type, name) ) and - gm.getFullyQualifiedNameWithTypes() = s2 + getFullyQualifiedNameWithTypes(gm) = s2 } diff --git a/csharp/ql/test/library-tests/overrides/Overrides22.ql b/csharp/ql/test/library-tests/overrides/Overrides22.ql index d2c5a9e4336..d6300d49ecd 100644 --- a/csharp/ql/test/library-tests/overrides/Overrides22.ql +++ b/csharp/ql/test/library-tests/overrides/Overrides22.ql @@ -1,4 +1,5 @@ import csharp +import semmle.code.csharp.commons.QualifiedName from Overridable v1, Overridable v2, string kind where @@ -9,4 +10,4 @@ where ) and v1.fromSource() and v2.fromSource() -select v1.getFullyQualifiedNameWithTypes(), v2.getFullyQualifiedNameWithTypes(), kind +select getFullyQualifiedNameWithTypes(v1), getFullyQualifiedNameWithTypes(v2), kind diff --git a/csharp/ql/test/library-tests/unification/Unification.ql b/csharp/ql/test/library-tests/unification/Unification.ql index 10c5e520921..f8c6c15377d 100644 --- a/csharp/ql/test/library-tests/unification/Unification.ql +++ b/csharp/ql/test/library-tests/unification/Unification.ql @@ -1,3 +1,4 @@ +import semmle.code.csharp.commons.QualifiedName import semmle.code.csharp.Unification class InterestingType extends @type { @@ -7,9 +8,9 @@ class InterestingType extends @type { } string toString() { - result = this.(Type).getFullyQualifiedNameWithTypes() + result = getFullyQualifiedNameWithTypes(this.(Type)) or - not exists(this.(Type).getFullyQualifiedNameWithTypes()) and + not exists(getFullyQualifiedNameWithTypes(this.(Type))) and result = this.(Type).toStringWithTypes() } From 8fbfafc1d7a18356d4577280b813a5bcfec768d0 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Wed, 13 Mar 2024 16:27:01 +0100 Subject: [PATCH 18/50] C#: Dont cache the deprecated getFullyQualifiedName predicate. --- csharp/ql/lib/semmle/code/csharp/Element.qll | 1 - 1 file changed, 1 deletion(-) diff --git a/csharp/ql/lib/semmle/code/csharp/Element.qll b/csharp/ql/lib/semmle/code/csharp/Element.qll index 38176acfd92..c44a092bb4f 100644 --- a/csharp/ql/lib/semmle/code/csharp/Element.qll +++ b/csharp/ql/lib/semmle/code/csharp/Element.qll @@ -118,7 +118,6 @@ class NamedElement extends Element, @named_element { * Unbound generic types, such as `IList`, are represented as * ``System.Collections.Generic.IList`1``. */ - cached deprecated final string getFullyQualifiedName() { exists(string qualifier, string name | this.hasFullyQualifiedName(qualifier, name) | if qualifier = "" then result = name else result = qualifier + "." + name From fc689efd1b1459e3da7771e46d364874d2544e97 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Mon, 8 Apr 2024 13:47:59 +0200 Subject: [PATCH 19/50] C#: Add debug version of the getFullyQualifiedName predicate. --- csharp/ql/lib/semmle/code/csharp/Element.qll | 27 ++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/csharp/ql/lib/semmle/code/csharp/Element.qll b/csharp/ql/lib/semmle/code/csharp/Element.qll index c44a092bb4f..f8cb018be68 100644 --- a/csharp/ql/lib/semmle/code/csharp/Element.qll +++ b/csharp/ql/lib/semmle/code/csharp/Element.qll @@ -124,6 +124,33 @@ class NamedElement extends Element, @named_element { ) } + /** + * INTERNAL: Do not use. + * + * This is intended for DEBUG ONLY. + * Constructing the fully qualified name for all elements in a large codebase + * puts severe stress on the string pool. + * + * Gets the fully qualified name of this element, for example the + * fully qualified name of `M` on line 3 is `N.C.M` in + * + * ```csharp + * namespace N { + * class C { + * void M(int i, string s) { } + * } + * } + * ``` + * + * Unbound generic types, such as `IList`, are represented as + * ``System.Collections.Generic.IList`1``. + */ + final string getFullyQualifiedNameDebug() { + exists(string qualifier, string name | this.hasFullyQualifiedName(qualifier, name) | + if qualifier = "" then result = name else result = qualifier + "." + name + ) + } + /** Holds if this element has the fully qualified name `qualifier`.`name`. */ cached predicate hasFullyQualifiedName(string qualifier, string name) { From c389611e5c5860637fcce06834c57e4d0c217d53 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Mon, 8 Apr 2024 17:07:48 +0100 Subject: [PATCH 20/50] C++: Add spurious dataflow test. --- .../dataflow-consistency.expected | 8 +++++++ .../dataflow-tests/test-source-sink.expected | 2 ++ .../dataflow/dataflow-tests/test.cpp | 24 ++++++++++++++++++- .../dataflow-tests/type-bugs.expected | 1 + 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/cpp/ql/test/library-tests/dataflow/dataflow-tests/dataflow-consistency.expected b/cpp/ql/test/library-tests/dataflow/dataflow-tests/dataflow-consistency.expected index fa6958d92ea..109f5ffebd1 100644 --- a/cpp/ql/test/library-tests/dataflow/dataflow-tests/dataflow-consistency.expected +++ b/cpp/ql/test/library-tests/dataflow/dataflow-tests/dataflow-consistency.expected @@ -33,6 +33,7 @@ argHasPostUpdate | test.cpp:67:29:67:35 | source1 | ArgumentNode is missing PostUpdateNode. | | test.cpp:813:19:813:35 | * ... | ArgumentNode is missing PostUpdateNode. | | test.cpp:848:23:848:25 | rpx | ArgumentNode is missing PostUpdateNode. | +| test.cpp:1057:19:1057:21 | * ... | ArgumentNode is missing PostUpdateNode. | postWithInFlow | BarrierGuard.cpp:49:6:49:6 | x [post update] | PostUpdateNode should not be the target of local flow. | | BarrierGuard.cpp:60:7:60:7 | x [post update] | PostUpdateNode should not be the target of local flow. | @@ -168,6 +169,13 @@ postWithInFlow | test.cpp:1045:9:1045:11 | ref arg buf | PostUpdateNode should not be the target of local flow. | | test.cpp:1051:5:1051:11 | content [post update] | PostUpdateNode should not be the target of local flow. | | test.cpp:1052:9:1052:9 | a [inner post update] | PostUpdateNode should not be the target of local flow. | +| test.cpp:1056:5:1056:7 | * ... [post update] | PostUpdateNode should not be the target of local flow. | +| test.cpp:1056:6:1056:7 | pp [inner post update] | PostUpdateNode should not be the target of local flow. | +| test.cpp:1062:53:1062:53 | p [inner post update] | PostUpdateNode should not be the target of local flow. | +| test.cpp:1072:3:1072:4 | * ... [post update] | PostUpdateNode should not be the target of local flow. | +| test.cpp:1072:4:1072:4 | p [inner post update] | PostUpdateNode should not be the target of local flow. | +| test.cpp:1073:3:1073:4 | * ... [post update] | PostUpdateNode should not be the target of local flow. | +| test.cpp:1073:4:1073:4 | p [inner post update] | PostUpdateNode should not be the target of local flow. | viableImplInCallContextTooLarge uniqueParameterNodeAtPosition uniqueParameterNodePosition diff --git a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test-source-sink.expected b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test-source-sink.expected index e03ee68b8a3..ada532793c4 100644 --- a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test-source-sink.expected +++ b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test-source-sink.expected @@ -310,6 +310,8 @@ irFlow | test.cpp:1021:18:1021:32 | *call to indirect_source | test.cpp:1027:19:1027:28 | *translated | | test.cpp:1021:18:1021:32 | *call to indirect_source | test.cpp:1031:19:1031:28 | *translated | | test.cpp:1045:14:1045:19 | call to source | test.cpp:1046:7:1046:10 | * ... | +| test.cpp:1061:15:1061:38 | *call to indirect_source | test.cpp:1057:19:1057:21 | ** ... | +| test.cpp:1072:8:1072:13 | call to source | test.cpp:1074:8:1074:9 | * ... | | true_upon_entry.cpp:9:11:9:16 | call to source | true_upon_entry.cpp:13:8:13:8 | x | | true_upon_entry.cpp:17:11:17:16 | call to source | true_upon_entry.cpp:21:8:21:8 | x | | true_upon_entry.cpp:27:9:27:14 | call to source | true_upon_entry.cpp:29:8:29:8 | x | diff --git a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp index b2bff6327c5..5e3b502468b 100644 --- a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp +++ b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp @@ -1050,4 +1050,26 @@ void flow_out_of_address_with_local_flow() { MyStruct a; a.content = nullptr; sink(&a); // $ SPURIOUS: ast -} \ No newline at end of file +} + +static void static_func_that_reassigns_pointer_before_sink(char** pp) { // $ ast-def=pp ir-def=*pp ir-def=**pp + *pp = ""; + indirect_sink(*pp); // $ SPURIOUS: ir +} + +void test_static_func_that_reassigns_pointer_before_sink() { + char* p = (char*)indirect_source(); + static_func_that_reassigns_pointer_before_sink(&p); +} + +void single_object_in_both_cases(bool b, int x, int y) { + int* p; + if(b) { + p = &x; + } else { + p = &y; + } + *p = source(); + *p = 0; + sink(*p); // $ SPURIOUS: ir +} diff --git a/cpp/ql/test/library-tests/dataflow/dataflow-tests/type-bugs.expected b/cpp/ql/test/library-tests/dataflow/dataflow-tests/type-bugs.expected index 6706d79e902..4d87c2da534 100644 --- a/cpp/ql/test/library-tests/dataflow/dataflow-tests/type-bugs.expected +++ b/cpp/ql/test/library-tests/dataflow/dataflow-tests/type-bugs.expected @@ -15,4 +15,5 @@ incorrectBaseType | test.cpp:848:23:848:25 | (reference dereference) | Expected 'Node.getType()' to be int, but it was int * | | test.cpp:854:10:854:36 | * ... | Expected 'Node.getType()' to be const int, but it was int | | test.cpp:867:10:867:30 | * ... | Expected 'Node.getType()' to be const int, but it was int | +| test.cpp:1062:52:1062:53 | *& ... | Expected 'Node.getType()' to be char, but it was char * | failures From b2002a981a6f6a9f2d69dfd38bfd9cc8c58f3eb0 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Mon, 8 Apr 2024 17:08:05 +0100 Subject: [PATCH 21/50] C++: Use the shared typeflow library to determine whether a pointer points to a buffer or an object. --- cpp/ql/lib/qlpack.yml | 1 + .../dataflow/internal/SsaInternalsCommon.qll | 7 +- .../cpp/ir/dataflow/internal/TypeFlow.qll | 259 ++++++++++++++++++ 3 files changed, 262 insertions(+), 5 deletions(-) create mode 100644 cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll diff --git a/cpp/ql/lib/qlpack.yml b/cpp/ql/lib/qlpack.yml index 8d15b6d142a..27529f55a29 100644 --- a/cpp/ql/lib/qlpack.yml +++ b/cpp/ql/lib/qlpack.yml @@ -9,6 +9,7 @@ dependencies: codeql/dataflow: ${workspace} codeql/rangeanalysis: ${workspace} codeql/ssa: ${workspace} + codeql/typeflow: ${workspace} codeql/tutorial: ${workspace} codeql/util: ${workspace} codeql/xml: ${workspace} diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternalsCommon.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternalsCommon.qll index 862070820f8..7e5b4de8122 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternalsCommon.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternalsCommon.qll @@ -6,6 +6,7 @@ private import DataFlowImplCommon as DataFlowImplCommon private import DataFlowUtil private import semmle.code.cpp.models.interfaces.PointerWrapper private import DataFlowPrivate +private import TypeFlow private import semmle.code.cpp.ir.ValueNumbering /** @@ -955,11 +956,7 @@ private module Cached { * Holds if the address computed by `operand` is guaranteed to write * to a specific address. */ - private predicate isCertainAddress(Operand operand) { - valueNumberOfOperand(operand).getAnInstruction() instanceof VariableAddressInstruction - or - operand.getType() instanceof Cpp::ReferenceType - } + private predicate isCertainAddress(Operand operand) { isPointerToSingleObject(operand.getDef()) } /** * Holds if `address` is a use of an SSA variable rooted at `base`, and the diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll new file mode 100644 index 00000000000..44f2e60daac --- /dev/null +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll @@ -0,0 +1,259 @@ +private import cpp +private import semmle.code.cpp.ir.IR +private import codeql.typeflow.TypeFlow + +private module Input implements TypeFlowInput { + private predicate hasExactSingleType(Instruction i) { + // The address of a variable is always a single object + i instanceof VariableAddressInstruction + or + // A reference always points to a always a single object + i.getResultLanguageType().hasUnspecifiedType(any(ReferenceType rt), false) + or + // `this` is never an array + i instanceof InitializeThisInstruction + or + // An allocation of a non-array object + exists(AllocationExpr alloc | alloc = i.getUnconvertedResultExpression() | + // i.e., `new int`; + alloc instanceof NewExpr + or + // i.e., `malloc(sizeof(int))` + exists(SizeofTypeOperator sizeOf | sizeOf = alloc.getSizeExpr() | + not sizeOf.getTypeOperand().getUnspecifiedType() instanceof ArrayType + ) + ) + } + + private predicate hasExactBufferType(Instruction i) { + // Anything with an array type is a buffer + i.getResultLanguageType().hasUnspecifiedType(any(ArrayType at), false) + or + not hasExactSingleType(i) and + i.getUnconvertedResultExpression() instanceof AllocationExpr + } + + private newtype TTypeFlowNode = + TInstructionNode(Instruction i) or + TFunctionNode(IRFunction func) + + abstract class TypeFlowNode extends TTypeFlowNode { + /** Gets a textual representation of this node. */ + abstract string toString(); + + /** + * Gets the type of this node. This type may not be the most precise + * possible type, but will be used as a starting point of the analysis. + */ + abstract Type getType(); + + /** Gets the location of this node. */ + abstract Location getLocation(); + + /** Gets the underlying `Instruction` of this node, if any. */ + Instruction asInstruction() { none() } + + /** Gets the underlying `IRFunction` of this node, if any. */ + IRFunction asFunction() { none() } + + /** Holds if the value of this node is always null. */ + abstract predicate isNullValue(); + } + + private class InstructionNode extends TypeFlowNode, TInstructionNode { + Instruction instr; + + InstructionNode() { this = TInstructionNode(instr) } + + override string toString() { result = instr.toString() } + + override Type getType() { + if hasExactSingleType(instr) then result.isSingle() else result.isBuffer() + } + + override Location getLocation() { result = instr.getLocation() } + + override Instruction asInstruction() { result = instr } + + override predicate isNullValue() { + instr.(ConstantInstruction).getValue() = "0" and + instr.getResultIRType() instanceof IRAddressType + } + } + + /** Gets the `TypeFlowNode` corresponding to `i`. */ + additional InstructionNode instructionNode(Instruction i) { result.asInstruction() = i } + + private class FunctionNode extends TypeFlowNode, TFunctionNode { + IRFunction func; + + FunctionNode() { this = TFunctionNode(func) } + + override string toString() { result = func.toString() } + + Instruction getReturnValueInstruction() { + result = func.getReturnInstruction().(ReturnValueInstruction).getReturnValue() + } + + override Type getType() { result = instructionNode(this.getReturnValueInstruction()).getType() } + + override Location getLocation() { result = func.getLocation() } + + override IRFunction asFunction() { result = func } + + override predicate isNullValue() { + instructionNode(this.getReturnValueInstruction()).isNullValue() + } + } + + /** + * Gets an ultimiate definition of `phi`. That is, an input to `phi` that is + * not itself a `PhiInstruction`. + */ + private Instruction getAnUltimateLocalDefinition(PhiInstruction phi) { + result = phi.getAnInput*() and not result instanceof PhiInstruction + } + + /** + * Holds if this function is private (i.e., cannot be accessed outside its + * compilation unit). This means we can use a closed-world assumption about + * calls to this function. + */ + private predicate isPrivate(Function func) { + func.isStatic() + or + func.getNamespace().getParentNamespace*().isInline() + or + func.(MemberFunction).isPrivate() + } + + /** + * Holds if `arg` is an argument for the parameter `p` in a private callable. + */ + pragma[nomagic] + private predicate privateParamArg(InitializeParameterInstruction p, Instruction arg) { + exists(CallInstruction call, int i, Function func | + call.getArgument(pragma[only_bind_into](i)) = arg and + func = call.getStaticCallTarget() and + func.getParameter(pragma[only_bind_into](i)) = p.getParameter() and + isPrivate(func) + ) + } + + predicate joinStep(TypeFlowNode n1, TypeFlowNode n2) { + // instruction -> phi + getAnUltimateLocalDefinition(n2.asInstruction()) = n1.asInstruction() + or + // return value -> function + n2.(FunctionNode).getReturnValueInstruction() = n1.asInstruction() + or + // function -> call + exists(Function func | func = n1.asFunction().getFunction() | + not func.isVirtual() and + n2.asInstruction().(CallInstruction).getStaticCallTarget() = func + ) + or + // Argument -> parameter where the parameter's enclosing function + // is "private". + exists(Instruction arg, Instruction p | + privateParamArg(p, arg) and + n1.asInstruction() = arg and + n2.asInstruction() = p + ) + } + + /** + * Holds if knowing whether `i1` points to a single object or buffer implies + * knowing whether `i2` points to a single object or buffer. + */ + private predicate instructionStep(Instruction i1, Instruction i2) { + i2.(CopyInstruction).getSourceValue() = i1 + or + i2.(CopyValueInstruction).getSourceValue() = i1 + or + i2.(ConvertInstruction).getUnary() = i1 + or + i2.(CheckedConvertOrNullInstruction).getUnary() = i1 + or + i2.(InheritanceConversionInstruction).getUnary() = i1 + or + i2.(PointerArithmeticInstruction).getLeft() = i1 + } + + predicate step(TypeFlowNode n1, TypeFlowNode n2) { + instructionStep(n1.asInstruction(), n2.asInstruction()) + } + + predicate isNullValue(TypeFlowNode n) { n.isNullValue() } + + private newtype TType = + TSingle() or + TBuffer() + + class Type extends TType { + string toString() { + this.isSingle() and + result = "Single" + or + this.isBuffer() and + result = "Buffer" + } + + /** Holds if this type is the type that represents a single object. */ + predicate isSingle() { this = TSingle() } + + /** Holds if this type is the type that represents a buffer. */ + predicate isBuffer() { this = TBuffer() } + + /** + * Gets a super type of this type, if any. + * + * The type relation is `Single <: Buffer`. + */ + Type getASupertype() { + this.isSingle() and + result.isBuffer() + } + } + + predicate exactTypeBase(TypeFlowNode n, Type t) { + exists(Instruction instr | instr = n.asInstruction() | + hasExactSingleType(instr) and t.isSingle() + or + hasExactBufferType(instr) and t.isBuffer() + ) + } + + pragma[nomagic] + private predicate upcastCand(TypeFlowNode n, Type t1, Type t2) { + exists(TypeFlowNode next | + step(n, next) + or + joinStep(n, next) + | + n.getType() = t1 and + next.getType() = t2 and + t1 != t2 + ) + } + + private predicate upcast(TypeFlowNode n, Type t1) { + exists(Type t2 | upcastCand(n, t1, t2) | + // No need for transitive closure since the subtyping relation is just `Single <: Buffer` + t1.getASupertype() = t2 + ) + } + + predicate typeFlowBaseCand(TypeFlowNode n, Type t) { upcast(n, t) } +} + +private module TypeFlow = Make; + +/** + * Holds if `i` is an instruction that computes an address that points to a + * single object (as opposed to pointing into a buffer). + */ +pragma[nomagic] +predicate isPointerToSingleObject(Instruction i) { + TypeFlow::bestTypeFlow(Input::instructionNode(i), any(Input::Type t | t.isSingle()), _) +} From 386580fc94750a0036d0b27e21ff05d86dacdb2c Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Mon, 8 Apr 2024 17:10:18 +0100 Subject: [PATCH 22/50] C++: Accept test changes. --- .../dataflow/dataflow-tests/test-source-sink.expected | 2 -- cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test-source-sink.expected b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test-source-sink.expected index ada532793c4..e03ee68b8a3 100644 --- a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test-source-sink.expected +++ b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test-source-sink.expected @@ -310,8 +310,6 @@ irFlow | test.cpp:1021:18:1021:32 | *call to indirect_source | test.cpp:1027:19:1027:28 | *translated | | test.cpp:1021:18:1021:32 | *call to indirect_source | test.cpp:1031:19:1031:28 | *translated | | test.cpp:1045:14:1045:19 | call to source | test.cpp:1046:7:1046:10 | * ... | -| test.cpp:1061:15:1061:38 | *call to indirect_source | test.cpp:1057:19:1057:21 | ** ... | -| test.cpp:1072:8:1072:13 | call to source | test.cpp:1074:8:1074:9 | * ... | | true_upon_entry.cpp:9:11:9:16 | call to source | true_upon_entry.cpp:13:8:13:8 | x | | true_upon_entry.cpp:17:11:17:16 | call to source | true_upon_entry.cpp:21:8:21:8 | x | | true_upon_entry.cpp:27:9:27:14 | call to source | true_upon_entry.cpp:29:8:29:8 | x | diff --git a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp index 5e3b502468b..af9e18034ed 100644 --- a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp +++ b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp @@ -1054,7 +1054,7 @@ void flow_out_of_address_with_local_flow() { static void static_func_that_reassigns_pointer_before_sink(char** pp) { // $ ast-def=pp ir-def=*pp ir-def=**pp *pp = ""; - indirect_sink(*pp); // $ SPURIOUS: ir + indirect_sink(*pp); // clean } void test_static_func_that_reassigns_pointer_before_sink() { @@ -1071,5 +1071,5 @@ void single_object_in_both_cases(bool b, int x, int y) { } *p = source(); *p = 0; - sink(*p); // $ SPURIOUS: ir + sink(*p); // clean } From e6984aa865e23a48215c7280258af247892db8b6 Mon Sep 17 00:00:00 2001 From: Tom Hvitved Date: Tue, 9 Apr 2024 10:10:25 +0200 Subject: [PATCH 23/50] Ruby: Remove two redundant `allowImplicitRead` predicates --- .../codeql/ruby/security/UnsafeCodeConstructionQuery.qll | 6 ------ .../ruby/security/UnsafeShellCommandConstructionQuery.qll | 6 ------ 2 files changed, 12 deletions(-) diff --git a/ruby/ql/lib/codeql/ruby/security/UnsafeCodeConstructionQuery.qll b/ruby/ql/lib/codeql/ruby/security/UnsafeCodeConstructionQuery.qll index 4cf57f36071..32cc9a4f821 100644 --- a/ruby/ql/lib/codeql/ruby/security/UnsafeCodeConstructionQuery.qll +++ b/ruby/ql/lib/codeql/ruby/security/UnsafeCodeConstructionQuery.qll @@ -46,12 +46,6 @@ private module UnsafeCodeConstructionConfig implements DataFlow::ConfigSig { // override to require the path doesn't have unmatched return steps DataFlow::FlowFeature getAFeature() { result instanceof DataFlow::FeatureHasSourceCallContext } - - predicate allowImplicitRead(DataFlow::Node node, DataFlow::ContentSet set) { - // allow implicit reads of array elements - isSink(node) and - set.isElementOfTypeOrUnknown("int") - } } /** diff --git a/ruby/ql/lib/codeql/ruby/security/UnsafeShellCommandConstructionQuery.qll b/ruby/ql/lib/codeql/ruby/security/UnsafeShellCommandConstructionQuery.qll index 7576702a2d4..b4e0b8b6bb5 100644 --- a/ruby/ql/lib/codeql/ruby/security/UnsafeShellCommandConstructionQuery.qll +++ b/ruby/ql/lib/codeql/ruby/security/UnsafeShellCommandConstructionQuery.qll @@ -49,12 +49,6 @@ private module UnsafeShellCommandConstructionConfig implements DataFlow::ConfigS // override to require the path doesn't have unmatched return steps DataFlow::FlowFeature getAFeature() { result instanceof DataFlow::FeatureHasSourceCallContext } - - predicate allowImplicitRead(DataFlow::Node node, DataFlow::ContentSet set) { - // allow implicit reads of array elements - isSink(node) and - set.isElementOfTypeOrUnknown("int") - } } /** From 10d96ee02f6fa0e84693135ffb830acc35d2cb8e Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Tue, 9 Apr 2024 10:31:48 +0200 Subject: [PATCH 24/50] C#: Address review comments. --- .../diag_recursive_generics/Types.ql | 7 +++---- csharp/ql/lib/semmle/code/csharp/Element.qll | 2 ++ .../ql/lib/semmle/code/csharp/dataflow/SSA.qll | 8 +------- .../attributes/AttributeElements.ql | 8 +++----- .../library-tests/constructors/Destructors1.ql | 3 +-- .../test/library-tests/csharp11/fileScoped.ql | 6 +----- .../ql/test/library-tests/csharp11/nativeInt.ql | 6 ++---- .../library-tests/csharp9/covariantReturn.ql | 13 ++++--------- csharp/ql/test/library-tests/csharp9/foreach.ql | 17 ++++++----------- csharp/ql/test/library-tests/csharp9/record.ql | 10 +--------- .../ql/test/library-tests/csharp9/withExpr.ql | 14 +++----------- csharp/ql/test/library-tests/enums/Enums3.ql | 8 +++----- .../ql/test/library-tests/generics/Generics.ql | 12 ++++-------- 13 files changed, 34 insertions(+), 80 deletions(-) diff --git a/csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql b/csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql index fff011dcbd5..de95e0fcbe7 100644 --- a/csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql +++ b/csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql @@ -1,6 +1,5 @@ import csharp -import semmle.code.csharp.commons.QualifiedName -from Class c, string qualifier, string name -where c.fromSource() and c.getBaseClass().hasFullyQualifiedName(qualifier, name) -select c, getQualifiedName(qualifier, name) +from Class c +where c.fromSource() +select c, c.getBaseClass().getFullyQualifiedNameDebug() diff --git a/csharp/ql/lib/semmle/code/csharp/Element.qll b/csharp/ql/lib/semmle/code/csharp/Element.qll index f8cb018be68..a48241a1408 100644 --- a/csharp/ql/lib/semmle/code/csharp/Element.qll +++ b/csharp/ql/lib/semmle/code/csharp/Element.qll @@ -145,6 +145,8 @@ class NamedElement extends Element, @named_element { * Unbound generic types, such as `IList`, are represented as * ``System.Collections.Generic.IList`1``. */ + bindingset[this] + pragma[inline_late] final string getFullyQualifiedNameDebug() { exists(string qualifier, string name | this.hasFullyQualifiedName(qualifier, name) | if qualifier = "" then result = name else result = qualifier + "." + name diff --git a/csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll b/csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll index 9cd33ea260f..0d79eafdf5c 100644 --- a/csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll +++ b/csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll @@ -3,7 +3,6 @@ */ import csharp -private import semmle.code.csharp.commons.QualifiedName /** * Provides classes for working with static single assignment (SSA) form. @@ -121,12 +120,7 @@ module Ssa { result = prefix + "." + this.getAssignable() | if f.(Modifiable).isStatic() - then - exists(string qualifier, string name | - f.getDeclaringType().hasFullyQualifiedName(qualifier, name) - | - prefix = getQualifiedName(qualifier, name) - ) + then prefix = f.getDeclaringType().getName() else prefix = "this" ) } diff --git a/csharp/ql/test/library-tests/attributes/AttributeElements.ql b/csharp/ql/test/library-tests/attributes/AttributeElements.ql index 17ce2ea3d93..679d7567ea5 100644 --- a/csharp/ql/test/library-tests/attributes/AttributeElements.ql +++ b/csharp/ql/test/library-tests/attributes/AttributeElements.ql @@ -1,9 +1,7 @@ import csharp -import semmle.code.csharp.commons.QualifiedName -from Attributable element, Attribute attribute, string qualifier, string name +from Attributable element, Attribute attribute where attribute = element.getAnAttribute() and - (attribute.fromSource() or element.(Assembly).getName() in ["attributes", "Assembly1"]) and - attribute.getType().hasFullyQualifiedName(qualifier, name) -select element, attribute, getQualifiedName(qualifier, name) + (attribute.fromSource() or element.(Assembly).getName() in ["attributes", "Assembly1"]) +select element, attribute, attribute.getType().getFullyQualifiedNameDebug() diff --git a/csharp/ql/test/library-tests/constructors/Destructors1.ql b/csharp/ql/test/library-tests/constructors/Destructors1.ql index 673a0e941f0..792d50da7bb 100644 --- a/csharp/ql/test/library-tests/constructors/Destructors1.ql +++ b/csharp/ql/test/library-tests/constructors/Destructors1.ql @@ -3,11 +3,10 @@ */ import csharp -import semmle.code.csharp.commons.QualifiedName from Destructor c, string qualifier, string name where c.getDeclaringType().hasFullyQualifiedName(qualifier, name) and qualifier = "Constructors" and name = "Class" -select c, getQualifiedName(qualifier, name) +select c, c.getDeclaringType().getFullyQualifiedNameDebug() diff --git a/csharp/ql/test/library-tests/csharp11/fileScoped.ql b/csharp/ql/test/library-tests/csharp11/fileScoped.ql index 3003fc801bf..33697255896 100644 --- a/csharp/ql/test/library-tests/csharp11/fileScoped.ql +++ b/csharp/ql/test/library-tests/csharp11/fileScoped.ql @@ -1,5 +1,4 @@ import csharp -private import semmle.code.csharp.commons.QualifiedName private predicate isInteresting(Type t) { ( @@ -20,10 +19,7 @@ query predicate typemodifiers(Type t, string modifier) { query predicate qualifiedtypes(Type t, string qualifiedName) { isInteresting(t) and - exists(string qualifier, string name | - t.hasFullyQualifiedName(qualifier, name) and - qualifiedName = getQualifiedName(qualifier, name) - ) + qualifiedName = t.getFullyQualifiedNameDebug() } query predicate filetypes(Type t) { diff --git a/csharp/ql/test/library-tests/csharp11/nativeInt.ql b/csharp/ql/test/library-tests/csharp11/nativeInt.ql index 80d7974de56..adbc062baf6 100644 --- a/csharp/ql/test/library-tests/csharp11/nativeInt.ql +++ b/csharp/ql/test/library-tests/csharp11/nativeInt.ql @@ -1,12 +1,10 @@ import csharp -import semmle.code.csharp.commons.QualifiedName -from LocalVariable v1, LocalVariable v2, Type t, string qualifier, string name +from LocalVariable v1, LocalVariable v2, Type t where v1.getFile().getStem() = "NativeInt" and v2.getFile().getStem() = "NativeInt" and t = v1.getType() and t = v2.getType() and - t.hasFullyQualifiedName(qualifier, name) and v1 != v2 -select v1, v2, getQualifiedName(qualifier, name) +select v1, v2, t.getFullyQualifiedNameDebug() diff --git a/csharp/ql/test/library-tests/csharp9/covariantReturn.ql b/csharp/ql/test/library-tests/csharp9/covariantReturn.ql index 6227ed18d6d..b4bab047322 100644 --- a/csharp/ql/test/library-tests/csharp9/covariantReturn.ql +++ b/csharp/ql/test/library-tests/csharp9/covariantReturn.ql @@ -1,13 +1,8 @@ import csharp -import semmle.code.csharp.commons.QualifiedName -from - Method m, Method overrider, string mnamespace, string mtype, string mname, string onamespace, - string otype, string oname +from Method m, Method overrider where m.getAnOverrider() = overrider and - m.getFile().getStem() = "CovariantReturn" and - m.hasFullyQualifiedName(mnamespace, mtype, mname) and - overrider.hasFullyQualifiedName(onamespace, otype, oname) -select getQualifiedName(mnamespace, mtype, mname), m.getReturnType().toString(), - getQualifiedName(onamespace, otype, oname), overrider.getReturnType().toString() + m.getFile().getStem() = "CovariantReturn" +select m.getFullyQualifiedNameDebug(), m.getReturnType().toString(), + overrider.getFullyQualifiedNameDebug(), overrider.getReturnType().toString() diff --git a/csharp/ql/test/library-tests/csharp9/foreach.ql b/csharp/ql/test/library-tests/csharp9/foreach.ql index cf6fcf2514a..343ecc556ab 100644 --- a/csharp/ql/test/library-tests/csharp9/foreach.ql +++ b/csharp/ql/test/library-tests/csharp9/foreach.ql @@ -1,5 +1,4 @@ import csharp -import semmle.code.csharp.commons.QualifiedName private string getLocation(Member m) { if m.fromSource() then result = m.getALocation().(SourceLocation).toString() else result = "-" @@ -9,13 +8,9 @@ private string getIsAsync(ForeachStmt f) { if f.isAsync() then result = "async" else result = "sync" } -from - ForeachStmt f, string qualifier1, string type1, string qualifier2, string type2, - string qualifier3, string type3 -where - f.getGetEnumerator().getDeclaringType().hasFullyQualifiedName(qualifier1, type1) and - f.getCurrent().getDeclaringType().hasFullyQualifiedName(qualifier2, type2) and - f.getMoveNext().getDeclaringType().hasFullyQualifiedName(qualifier3, type3) -select f, f.getElementType().toString(), getIsAsync(f), getQualifiedName(qualifier1, type1), - getLocation(f.getGetEnumerator()), getQualifiedName(qualifier2, type2), - getLocation(f.getCurrent()), getQualifiedName(qualifier3, type3), getLocation(f.getMoveNext()) +from ForeachStmt f +select f, f.getElementType().toString(), getIsAsync(f), + f.getGetEnumerator().getDeclaringType().getFullyQualifiedNameDebug(), + getLocation(f.getGetEnumerator()), f.getCurrent().getDeclaringType().getFullyQualifiedNameDebug(), + getLocation(f.getCurrent()), f.getMoveNext().getDeclaringType().getFullyQualifiedNameDebug(), + getLocation(f.getMoveNext()) diff --git a/csharp/ql/test/library-tests/csharp9/record.ql b/csharp/ql/test/library-tests/csharp9/record.ql index a2a9a9c0786..58cf579bac6 100644 --- a/csharp/ql/test/library-tests/csharp9/record.ql +++ b/csharp/ql/test/library-tests/csharp9/record.ql @@ -7,18 +7,10 @@ query predicate records(RecordClass t, string i, RecordCloneMethod clone) { t.fromSource() } -private string getMemberName(Member m) { - exists(string qualifier, string name | - m.getDeclaringType().hasFullyQualifiedName(qualifier, name) - | - result = getQualifiedName(qualifier, name) + "." + m.toStringWithTypes() - ) -} - query predicate members(RecordClass t, string ms, string l) { t.fromSource() and exists(Member m | t.hasMember(m) | - ms = getMemberName(m) and + ms = getFullyQualifiedNameWithTypes(m) and if m.fromSource() then l = m.getLocation().toString() else l = "no location" ) } diff --git a/csharp/ql/test/library-tests/csharp9/withExpr.ql b/csharp/ql/test/library-tests/csharp9/withExpr.ql index 6683d7c54f6..564cbe529aa 100644 --- a/csharp/ql/test/library-tests/csharp9/withExpr.ql +++ b/csharp/ql/test/library-tests/csharp9/withExpr.ql @@ -1,19 +1,11 @@ import csharp import semmle.code.csharp.commons.QualifiedName -private string getSignature(Method m) { - exists(string qualifier, string name | - m.getDeclaringType().hasFullyQualifiedName(qualifier, name) - | - result = getQualifiedName(qualifier, name) + "." + m.toStringWithTypes() - ) -} - query predicate withExpr(WithExpr with, string type, Expr expr, ObjectInitializer init, string clone) { type = with.getType().toStringWithTypes() and expr = with.getExpr() and init = with.getInitializer() and - clone = getSignature(with.getCloneMethod()) + clone = getFullyQualifiedNameWithTypes(with.getCloneMethod()) } query predicate withTarget(WithExpr with, RecordCloneMethod clone, Constructor ctor) { @@ -25,7 +17,7 @@ query predicate cloneOverrides(string b, string o) { exists(RecordCloneMethod base, RecordCloneMethod overrider | base.getDeclaringType().fromSource() and base.getAnOverrider() = overrider and - b = getSignature(base) and - o = getSignature(overrider) + b = getFullyQualifiedNameWithTypes(base) and + o = getFullyQualifiedNameWithTypes(overrider) ) } diff --git a/csharp/ql/test/library-tests/enums/Enums3.ql b/csharp/ql/test/library-tests/enums/Enums3.ql index 5cfbdc56193..b01ffb97e18 100644 --- a/csharp/ql/test/library-tests/enums/Enums3.ql +++ b/csharp/ql/test/library-tests/enums/Enums3.ql @@ -3,13 +3,11 @@ */ import csharp -import semmle.code.csharp.commons.QualifiedName -from EnumConstant c, string namespace, string name +from EnumConstant c where c.getName() = "Green" and c.getDeclaringType().hasFullyQualifiedName("Enums", "LongColor") and c.getType() = c.getDeclaringType() and - c.getValue() = "1" and - c.getDeclaringType().getBaseClass().hasFullyQualifiedName(namespace, name) -select c, getQualifiedName(namespace, name) + c.getValue() = "1" +select c, c.getDeclaringType().getBaseClass().getFullyQualifiedNameDebug() diff --git a/csharp/ql/test/library-tests/generics/Generics.ql b/csharp/ql/test/library-tests/generics/Generics.ql index 71e21a5815e..e75f1fdc908 100644 --- a/csharp/ql/test/library-tests/generics/Generics.ql +++ b/csharp/ql/test/library-tests/generics/Generics.ql @@ -268,16 +268,12 @@ query predicate test33(ConstructedMethod cm, string s1, string s2) { query predicate test34(UnboundGeneric ug, string s1, string s2) { ug.fromSource() and - exists(string qualifier, string name | - ug.hasFullyQualifiedName(qualifier, name) and s1 = getQualifiedName(qualifier, name) - ) and - getFullyQualifiedNameWithTypes(ug) = s2 + s1 = ug.getFullyQualifiedNameDebug() and + s2 = getFullyQualifiedNameWithTypes(ug) } query predicate test35(UnboundGenericMethod gm, string s1, string s2) { gm.fromSource() and - exists(string namespace, string type, string name | - gm.hasFullyQualifiedName(namespace, type, name) and s1 = getQualifiedName(namespace, type, name) - ) and - getFullyQualifiedNameWithTypes(gm) = s2 + s1 = gm.getFullyQualifiedNameDebug() and + s2 = getFullyQualifiedNameWithTypes(gm) } From bb4952f5575e71c19473982a0287db8773b4fd5a Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 9 Apr 2024 14:00:25 +0200 Subject: [PATCH 25/50] Revert "Python: Disable failing integration tests" This reverts commit 8c2455fc113f3d0ee91b5d264c96782e06a6c274. --- .../force-enable-library-extraction/{disabled-test.sh => test.sh} | 0 .../ignore-venv/{disabled-test.sh => test.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename python/extractor/cli-integration-test/force-enable-library-extraction/{disabled-test.sh => test.sh} (100%) rename python/extractor/cli-integration-test/ignore-venv/{disabled-test.sh => test.sh} (100%) diff --git a/python/extractor/cli-integration-test/force-enable-library-extraction/disabled-test.sh b/python/extractor/cli-integration-test/force-enable-library-extraction/test.sh similarity index 100% rename from python/extractor/cli-integration-test/force-enable-library-extraction/disabled-test.sh rename to python/extractor/cli-integration-test/force-enable-library-extraction/test.sh diff --git a/python/extractor/cli-integration-test/ignore-venv/disabled-test.sh b/python/extractor/cli-integration-test/ignore-venv/test.sh similarity index 100% rename from python/extractor/cli-integration-test/ignore-venv/disabled-test.sh rename to python/extractor/cli-integration-test/ignore-venv/test.sh From a0d6324f685ccfbada6d682379101f6822dfef44 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 9 Apr 2024 14:01:10 +0200 Subject: [PATCH 26/50] Python: Fix ignore-venv integration test Now that we no longer support the fallback option (https://github.com/github/codeql/pull/16127) --- .../cli-integration-test/ignore-venv/test.sh | 32 +------------------ 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/python/extractor/cli-integration-test/ignore-venv/test.sh b/python/extractor/cli-integration-test/ignore-venv/test.sh index e1368008891..8735203ec32 100755 --- a/python/extractor/cli-integration-test/ignore-venv/test.sh +++ b/python/extractor/cli-integration-test/ignore-venv/test.sh @@ -17,63 +17,33 @@ mkdir dbs # set up venvs cd repo_dir +# make venv with some package in it (so we show that our ignore logic is correct) python3 -m venv venv venv/bin/pip install flask -python3 -m venv venv2 - cd "$SCRIPTDIR" -# In 2.16.0 we stop extracting libraries by default, so to test this functionality we -# need to force enable it. Once we release 2.17.0 and turn off library extraction for -# good, we can remove the part of this test ensuring that dependencies in an active -# venv are still extracted (since that will no longer be the case). -export CODEQL_EXTRACTOR_PYTHON_FORCE_ENABLE_LIBRARY_EXTRACTION_UNTIL_2_17_0=1 - -# Create DBs with venv2 active (that does not have flask installed) -source repo_dir/venv2/bin/activate - export CODEQL_EXTRACTOR_PYTHON_DISABLE_AUTOMATIC_VENV_EXCLUDE= $CODEQL database create dbs/normal --language python --source-root repo_dir/ export CODEQL_EXTRACTOR_PYTHON_DISABLE_AUTOMATIC_VENV_EXCLUDE=1 $CODEQL database create dbs/no-venv-ignore --language python --source-root repo_dir/ -# Create DB with venv active that has flask installed. We want to ensure that we're -# still able to resolve imports to flask, but don't want to extract EVERYTHING from -# within the venv. Important note is that the test-file in the repo_dir actually imports -# flask :D -source repo_dir/venv/bin/activate -export CODEQL_EXTRACTOR_PYTHON_DISABLE_AUTOMATIC_VENV_EXCLUDE= -$CODEQL database create dbs/normal-with-flask-venv --language python --source-root repo_dir/ - # --- set +x EXTRACTED_NORMAL=$(unzip -l dbs/normal/src.zip | wc -l) EXTRACTED_NO_VENV_IGNORE=$(unzip -l dbs/no-venv-ignore/src.zip | wc -l) -EXTRACTED_ACTIVE_FLASK=$(unzip -l dbs/normal-with-flask-venv/src.zip | wc -l) exitcode=0 echo "EXTRACTED_NORMAL=$EXTRACTED_NORMAL" echo "EXTRACTED_NO_VENV_IGNORE=$EXTRACTED_NO_VENV_IGNORE" -echo "EXTRACTED_ACTIVE_FLASK=$EXTRACTED_ACTIVE_FLASK" if [[ ! $EXTRACTED_NORMAL -lt $EXTRACTED_NO_VENV_IGNORE ]]; then echo "ERROR: EXTRACTED_NORMAL not smaller EXTRACTED_NO_VENV_IGNORE" exitcode=1 fi -if [[ ! $EXTRACTED_NORMAL -lt $EXTRACTED_ACTIVE_FLASK ]]; then - echo "ERROR: EXTRACTED_NORMAL not smaller EXTRACTED_ACTIVE_FLASK" - exitcode=1 -fi - -if [[ ! $EXTRACTED_ACTIVE_FLASK -lt $EXTRACTED_NO_VENV_IGNORE ]]; then - echo "ERROR: EXTRACTED_ACTIVE_FLASK not smaller EXTRACTED_NO_VENV_IGNORE" - exitcode=1 -fi - exit $exitcode From e9e7ccddcea1f00326522457c5a5d816564d55df Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 9 Apr 2024 14:02:34 +0200 Subject: [PATCH 27/50] Python: delete `force-enable-library-extraction` integration test --- .../repo_dir/foo.py | 3 -- .../force-enable-library-extraction/test.sh | 41 ------------------- 2 files changed, 44 deletions(-) delete mode 100644 python/extractor/cli-integration-test/force-enable-library-extraction/repo_dir/foo.py delete mode 100755 python/extractor/cli-integration-test/force-enable-library-extraction/test.sh diff --git a/python/extractor/cli-integration-test/force-enable-library-extraction/repo_dir/foo.py b/python/extractor/cli-integration-test/force-enable-library-extraction/repo_dir/foo.py deleted file mode 100644 index cf0cd77a108..00000000000 --- a/python/extractor/cli-integration-test/force-enable-library-extraction/repo_dir/foo.py +++ /dev/null @@ -1,3 +0,0 @@ -import pip - -print(42) diff --git a/python/extractor/cli-integration-test/force-enable-library-extraction/test.sh b/python/extractor/cli-integration-test/force-enable-library-extraction/test.sh deleted file mode 100755 index 9d74cfaca4b..00000000000 --- a/python/extractor/cli-integration-test/force-enable-library-extraction/test.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -set -Eeuo pipefail # see https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ - -set -x - -CODEQL=${CODEQL:-codeql} - -SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$SCRIPTDIR" - -# start on clean slate -rm -rf dbs -mkdir dbs - -cd "$SCRIPTDIR" - -export CODEQL_EXTRACTOR_PYTHON_FORCE_ENABLE_LIBRARY_EXTRACTION_UNTIL_2_17_0= -$CODEQL database create dbs/normal --language python --source-root repo_dir/ - -export CODEQL_EXTRACTOR_PYTHON_FORCE_ENABLE_LIBRARY_EXTRACTION_UNTIL_2_17_0=1 -$CODEQL database create dbs/with-lib-extraction --language python --source-root repo_dir/ - -# --- - -set +x - -EXTRACTED_NORMAL=$(unzip -l dbs/normal/src.zip | wc -l) -EXTRACTED_WITH_LIB_EXTRACTION=$(unzip -l dbs/with-lib-extraction/src.zip | wc -l) - -exitcode=0 - -echo "EXTRACTED_NORMAL=$EXTRACTED_NORMAL" -echo "EXTRACTED_WITH_LIB_EXTRACTION=$EXTRACTED_WITH_LIB_EXTRACTION" - -if [[ ! $EXTRACTED_WITH_LIB_EXTRACTION -gt $EXTRACTED_NORMAL ]]; then - echo "ERROR: EXTRACTED_WITH_LIB_EXTRACTION not greater than EXTRACTED_NORMAL" - exitcode=1 -fi - -exit $exitcode From ae8240a695b532778f8ef8f4864f626744cfc705 Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Tue, 9 Apr 2024 13:23:03 +0100 Subject: [PATCH 28/50] Delete models for JDK internal packages --- .../jdk.internal.access.foreign.model.yml | 10 - .../generated/jdk.internal.access.model.yml | 243 ------------------ .../ext/generated/jdk.internal.misc.model.yml | 11 - .../jdk.internal.org.objectweb.asm.model.yml | 15 -- 4 files changed, 279 deletions(-) delete mode 100644 java/ql/lib/ext/generated/jdk.internal.access.foreign.model.yml delete mode 100644 java/ql/lib/ext/generated/jdk.internal.access.model.yml delete mode 100644 java/ql/lib/ext/generated/jdk.internal.misc.model.yml delete mode 100644 java/ql/lib/ext/generated/jdk.internal.org.objectweb.asm.model.yml diff --git a/java/ql/lib/ext/generated/jdk.internal.access.foreign.model.yml b/java/ql/lib/ext/generated/jdk.internal.access.foreign.model.yml deleted file mode 100644 index 5cfddbd9303..00000000000 --- a/java/ql/lib/ext/generated/jdk.internal.access.foreign.model.yml +++ /dev/null @@ -1,10 +0,0 @@ -# THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT. -extensions: - - addsTo: - pack: codeql/java-all - extensible: neutralModel - data: - - ["jdk.internal.access.foreign", "UnmapperProxy", "address", "()", "summary", "df-generated"] - - ["jdk.internal.access.foreign", "UnmapperProxy", "fileDescriptor", "()", "summary", "df-generated"] - - ["jdk.internal.access.foreign", "UnmapperProxy", "isSync", "()", "summary", "df-generated"] - - ["jdk.internal.access.foreign", "UnmapperProxy", "unmap", "()", "summary", "df-generated"] diff --git a/java/ql/lib/ext/generated/jdk.internal.access.model.yml b/java/ql/lib/ext/generated/jdk.internal.access.model.yml deleted file mode 100644 index 37d80349303..00000000000 --- a/java/ql/lib/ext/generated/jdk.internal.access.model.yml +++ /dev/null @@ -1,243 +0,0 @@ -# THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT. -extensions: - - addsTo: - pack: codeql/java-all - extensible: summaryModel - data: - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", True, "registerCleanup", "(FileDescriptor,PhantomCleanable)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaIOFilePermissionAccess", True, "newPermPlusAltPath", "(FilePermission)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaIOFilePermissionAccess", True, "newPermUsingAltPath", "(FilePermission)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaIORandomAccessFileAccess", True, "openAndDelete", "(File,String)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "addEnableNativeAccess", "(Module)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "addOpensToAllUnnamed", "(Module,Set,Set)", "", "Argument[1].Element", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "addOpensToAllUnnamed", "(Module,Set,Set)", "", "Argument[2].Element", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "createOrGetClassLoaderValueMap", "(ClassLoader)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "defineModule", "(ClassLoader,ModuleDescriptor,URI)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "defineModule", "(ClassLoader,ModuleDescriptor,URI)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "definePackage", "(ClassLoader,String,Module)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "definePackage", "(ClassLoader,String,Module)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "defineUnnamedModule", "(ClassLoader)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "getBytesNoRepl", "(String,Charset)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "getBytesUTF8NoRepl", "(String)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "getServicesCatalog", "(ModuleLayer)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "join", "(String,String,String,String[],int)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "join", "(String,String,String,String[],int)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "join", "(String,String,String,String[],int)", "", "Argument[2]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "join", "(String,String,String,String[],int)", "", "Argument[3].ArrayElement", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "layers", "(ClassLoader)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "layers", "(ModuleLayer)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "newStringNoRepl", "(byte[],Charset)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "newStringUTF8NoRepl", "(byte[],int,int)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "newThreadWithAcc", "(Runnable,AccessControlContext)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "newThreadWithAcc", "(Runnable,AccessControlContext)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "setCause", "(Throwable,Throwable)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "stringConcatHelper", "(String,MethodType)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "stringConcatHelper", "(String,MethodType)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "collectCoordinates", "(VarHandle,int,MethodHandle)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "dropCoordinates", "(VarHandle,int,Class[])", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "filterCoordinates", "(VarHandle,int,MethodHandle[])", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "filterValue", "(VarHandle,MethodHandle,MethodHandle)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "getMethodDescriptor", "(Object)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "getMethodType", "(Object)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "getName", "(Object)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "insertCoordinates", "(VarHandle,int,Object[])", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "nativeMethodHandle", "(NativeEntryPoint,MethodHandle)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "nativeMethodHandle", "(NativeEntryPoint,MethodHandle)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "permuteCoordinates", "(VarHandle,List,int[])", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "permuteCoordinates", "(VarHandle,List,int[])", "", "Argument[1].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newConfiguration", "(ModuleFinder,Map)", "", "Argument[1].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newExports", "(Set,String)", "", "Argument[0].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newExports", "(Set,String)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newExports", "(Set,String,Set)", "", "Argument[0].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newExports", "(Set,String,Set)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newExports", "(Set,String,Set)", "", "Argument[2].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleBuilder", "(String,boolean,Set)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleBuilder", "(String,boolean,Set)", "", "Argument[2].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[2].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[3].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[4].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[5].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[6].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[7].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[8].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[9]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newOpens", "(Set,String)", "", "Argument[0].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newOpens", "(Set,String)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newOpens", "(Set,String,Set)", "", "Argument[0].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newOpens", "(Set,String,Set)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newOpens", "(Set,String,Set)", "", "Argument[2].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newProvides", "(String,List)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newProvides", "(String,List)", "", "Argument[1].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newRequires", "(Set,String,ModuleDescriptor$Version)", "", "Argument[0].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newRequires", "(Set,String,ModuleDescriptor$Version)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newRequires", "(Set,String,ModuleDescriptor$Version)", "", "Argument[2]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "packages", "(ModuleDescriptor$Builder)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "copyConstructor", "(Constructor)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "copyField", "(Field)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "copyMethod", "(Method)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "getConstructorAccessor", "(Constructor)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "getConstructorAnnotations", "(Constructor)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "getConstructorParameterAnnotations", "(Constructor)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "getConstructorSignature", "(Constructor)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "getExecutableSharedParameterTypes", "(Executable)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "getMethodAccessor", "(Method)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "getRoot", "(AccessibleObject)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "leafCopyMethod", "(Method)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "newConstructor", "(Class,Class[],Class[],int,int,String,byte[],byte[])", "", "Argument[1].ArrayElement", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "newConstructor", "(Class,Class[],Class[],int,int,String,byte[],byte[])", "", "Argument[2].ArrayElement", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "newConstructor", "(Class,Class[],Class[],int,int,String,byte[],byte[])", "", "Argument[5]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "newConstructor", "(Class,Class[],Class[],int,int,String,byte[],byte[])", "", "Argument[6]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "newConstructor", "(Class,Class[],Class[],int,int,String,byte[],byte[])", "", "Argument[7]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "setConstructorAccessor", "(Constructor,ConstructorAccessor)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "setMethodAccessor", "(Method,MethodAccessor)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetHttpCookieAccess", True, "header", "(HttpCookie)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetHttpCookieAccess", True, "parse", "(String)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetInetAddressAccess", True, "addressBytes", "(Inet6Address)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetInetAddressAccess", True, "getByName", "(String,InetAddress)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetInetAddressAccess", True, "getOriginalHostName", "(InetAddress)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetURLAccess", True, "getHandler", "(URL)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetUriAccess", True, "create", "(String,String)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetUriAccess", True, "create", "(String,String)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "acquireScope", "(Buffer,boolean)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "bufferSegment", "(Buffer)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "getBufferBase", "(ByteBuffer)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "newDirectByteBuffer", "(long,int,Object,MemorySegmentProxy)", "", "Argument[2]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "newDirectByteBuffer", "(long,int,Object,MemorySegmentProxy)", "", "Argument[3]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "newHeapByteBuffer", "(byte[],int,int,MemorySegmentProxy)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "newHeapByteBuffer", "(byte[],int,int,MemorySegmentProxy)", "", "Argument[3]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "newMappedByteBuffer", "(UnmapperProxy,long,int,Object,MemorySegmentProxy)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "newMappedByteBuffer", "(UnmapperProxy,long,int,Object,MemorySegmentProxy)", "", "Argument[3]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "newMappedByteBuffer", "(UnmapperProxy,long,int,Object,MemorySegmentProxy)", "", "Argument[4]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecurityAccess$ProtectionDomainCache", True, "get", "(ProtectionDomain)", "", "Argument[this]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecurityAccess$ProtectionDomainCache", True, "put", "(ProtectionDomain,PermissionCollection)", "", "Argument[0]", "Argument[this]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecurityAccess$ProtectionDomainCache", True, "put", "(ProtectionDomain,PermissionCollection)", "", "Argument[1]", "Argument[this]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecurityAccess", True, "doIntersectionPrivilege", "(PrivilegedAction,AccessControlContext)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecurityAccess", True, "doIntersectionPrivilege", "(PrivilegedAction,AccessControlContext,AccessControlContext)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecurityAccess", True, "getProtectDomains", "(AccessControlContext)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecuritySignatureAccess", True, "initSign", "(Signature,PrivateKey,AlgorithmParameterSpec,SecureRandom)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecuritySignatureAccess", True, "initSign", "(Signature,PrivateKey,AlgorithmParameterSpec,SecureRandom)", "", "Argument[2]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecuritySignatureAccess", True, "initSign", "(Signature,PrivateKey,AlgorithmParameterSpec,SecureRandom)", "", "Argument[3]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecuritySignatureAccess", True, "initVerify", "(Signature,PublicKey,AlgorithmParameterSpec)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecuritySignatureAccess", True, "initVerify", "(Signature,PublicKey,AlgorithmParameterSpec)", "", "Argument[2]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilCollectionAccess", True, "listFromTrustedArray", "(Object[])", "", "Argument[0].ArrayElement", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilCollectionAccess", True, "listFromTrustedArrayNullsAllowed", "(Object[])", "", "Argument[0].ArrayElement", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "entryFor", "(JarFile,String)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "entryNames", "(JarFile,CodeSource[])", "", "Argument[1].ArrayElement", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getCodeSource", "(JarFile,URL,String)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getCodeSource", "(JarFile,URL,String)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getCodeSource", "(JarFile,URL,String)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getCodeSource", "(JarFile,URL,String)", "", "Argument[2]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getCodeSources", "(JarFile,URL)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getCodeSources", "(JarFile,URL)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getCodeSources", "(JarFile,URL)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getManifestDigests", "(JarFile)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getTrustedAttributes", "(Manifest,String)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilResourceBundleAccess", True, "getBundle", "(String,Locale,Module)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilResourceBundleAccess", True, "getBundle", "(String,Locale,Module)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilResourceBundleAccess", True, "getParent", "(ResourceBundle)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilResourceBundleAccess", True, "setLocale", "(ResourceBundle,Locale)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilResourceBundleAccess", True, "setName", "(ResourceBundle,String)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilResourceBundleAccess", True, "setParent", "(ResourceBundle,ResourceBundle)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - addsTo: - pack: codeql/java-all - extensible: neutralModel - data: - - ["jdk.internal.access", "JavaBeansAccess", "getConstructorPropertiesValue", "(Constructor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaBeansAccess", "getReadMethod", "(Class,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOAccess", "charset", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOAccess", "console", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "close", "(FileDescriptor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "get", "(FileDescriptor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "getAppend", "(FileDescriptor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "getHandle", "(FileDescriptor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "registerCleanup", "(FileDescriptor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "set", "(FileDescriptor,int)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "setAppend", "(FileDescriptor,boolean)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "setHandle", "(FileDescriptor,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "unregisterCleanup", "(FileDescriptor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addEnableNativeAccessAllUnnamed", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addExports", "(Module,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addExports", "(Module,String,Module)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addExportsToAllUnnamed", "(Module,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addNonExportedPackages", "(ModuleLayer)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addOpens", "(Module,String,Module)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addOpensToAllUnnamed", "(Module,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addReads", "(Module,Module)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addReadsAllUnnamed", "(Module)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addUses", "(Module,Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "bindToLoader", "(ModuleLayer,ClassLoader)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "blockedOn", "(Interruptible)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "casAnnotationType", "(Class,AnnotationType,AnnotationType)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "classData", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "decodeASCII", "(byte[],int,char[],int,int)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "defineClass", "(ClassLoader,Class,String,byte[],ProtectionDomain,boolean,int,Object)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "defineClass", "(ClassLoader,String,byte[],ProtectionDomain,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "exit", "(int)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "fastUUID", "(long,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "findBootstrapClassOrNull", "(String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "findNative", "(ClassLoader,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getAnnotationType", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getConstantPool", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getDeclaredAnnotationMap", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getDeclaredPublicMethods", "(Class,String,Class[])", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getEnumConstantsShared", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getRawClassAnnotations", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getRawClassTypeAnnotations", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getRawExecutableTypeAnnotations", "(Executable)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "inflateBytesToChars", "(byte[],int,char[],int,int)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "invalidatePackageAccessCache", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "invokeFinalize", "(Object)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "isEnableNativeAccess", "(Module)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "isReflectivelyExported", "(Module,String,Module)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "isReflectivelyOpened", "(Module,String,Module)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "protectionDomain", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "registerShutdownHook", "(int,boolean,Runnable)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "stringConcatInitialCoder", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "stringConcatMix", "(long,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", "ensureCustomized", "(MethodHandle)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", "generateHolderClasses", "(Stream)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", "getDeclaringClass", "(Object)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", "isNative", "(Object)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", "memoryAccessVarHandle", "(Class,boolean,long,ByteOrder)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", "newMemberName", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", "requires", "(ModuleDescriptor$Builder,Set,String,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", "resolveAndBind", "(ModuleFinder,Collection,PrintStream)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangRefAccess", "runFinalization", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangRefAccess", "waitForReferenceProcessing", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", "getConstructorSlot", "(Constructor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", "getExecutableTypeAnnotationBytes", "(Executable)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", "isTrustedFinalField", "(Field)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", "newInstance", "(Constructor,Object[],Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNetInetAddressAccess", "addressValue", "(Inet4Address)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "force", "(FileDescriptor,long,boolean,long,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "getBufferAddress", "(ByteBuffer)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "getDirectBufferPool", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "isLoaded", "(long,boolean,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "load", "(long,boolean,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "pageSize", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "reserveMemory", "(long,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "unload", "(long,boolean,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "unmapper", "(ByteBuffer)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "unreserveMemory", "(long,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaSecurityAccess", "getProtectionDomainCache", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaSecuritySignatureAccess", "initVerify", "(Signature,Certificate,AlgorithmParameterSpec)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaSecuritySpecAccess", "clearEncodedKeySpec", "(EncodedKeySpec)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", "ensureInitialization", "(JarFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", "entries2", "(JarFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", "isInitializing", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", "jarFileHasClassPathAttribute", "(JarFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", "setEagerValidation", "(JarFile,boolean)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilResourceBundleAccess", "newResourceBundle", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "entries", "(ZipFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "entryNameStream", "(ZipFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "getExtraAttributes", "(ZipEntry)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "getManifestAndSignatureRelatedFiles", "(JarFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "getManifestName", "(JarFile,boolean)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "getManifestNum", "(JarFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "getMetaInfVersions", "(JarFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "setExtraAttributes", "(ZipEntry,int)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "startsWithLocHeader", "(ZipFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "stream", "(ZipFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaxCryptoSpecAccess", "clearSecretKeySpec", "(SecretKeySpec)", "summary", "df-generated"] diff --git a/java/ql/lib/ext/generated/jdk.internal.misc.model.yml b/java/ql/lib/ext/generated/jdk.internal.misc.model.yml deleted file mode 100644 index 3ab4816a586..00000000000 --- a/java/ql/lib/ext/generated/jdk.internal.misc.model.yml +++ /dev/null @@ -1,11 +0,0 @@ -# THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT. -extensions: - - addsTo: - pack: codeql/java-all - extensible: neutralModel - data: - - ["jdk.internal.misc", "Signal$Handler", "handle", "(Signal)", "summary", "df-generated"] - - ["jdk.internal.misc", "VM$BufferPool", "getCount", "()", "summary", "df-generated"] - - ["jdk.internal.misc", "VM$BufferPool", "getMemoryUsed", "()", "summary", "df-generated"] - - ["jdk.internal.misc", "VM$BufferPool", "getName", "()", "summary", "df-generated"] - - ["jdk.internal.misc", "VM$BufferPool", "getTotalCapacity", "()", "summary", "df-generated"] diff --git a/java/ql/lib/ext/generated/jdk.internal.org.objectweb.asm.model.yml b/java/ql/lib/ext/generated/jdk.internal.org.objectweb.asm.model.yml deleted file mode 100644 index 1121dda6781..00000000000 --- a/java/ql/lib/ext/generated/jdk.internal.org.objectweb.asm.model.yml +++ /dev/null @@ -1,15 +0,0 @@ -# THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT. -extensions: - - addsTo: - pack: codeql/java-all - extensible: summaryModel - data: - - ["jdk.internal.org.objectweb.asm", "ClassVisitor", True, "visitAnnotation", "(String,boolean)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.org.objectweb.asm", "ClassVisitor", True, "visitAnnotation", "(String,boolean)", "", "Argument[this]", "ReturnValue", "taint", "df-generated"] - - addsTo: - pack: codeql/java-all - extensible: neutralModel - data: - - ["jdk.internal.org.objectweb.asm", "ClassVisitor", "visit", "(int,int,String,String,String,String[])", "summary", "df-generated"] - - ["jdk.internal.org.objectweb.asm", "ClassVisitor", "visitAttribute", "(Attribute)", "summary", "df-generated"] - - ["jdk.internal.org.objectweb.asm", "ClassVisitor", "visitModule", "(String,int,String)", "summary", "df-generated"] From e6d63b980d044c167d8cfa5f60117c3cbc82c633 Mon Sep 17 00:00:00 2001 From: Felicity Chapman Date: Tue, 9 Apr 2024 14:05:42 +0100 Subject: [PATCH 29/50] Make duplicated content clear --- .../customizing-library-models-for-csharp.rst | 15 +++------------ ...omizing-library-models-for-java-and-kotlin.rst | 11 +---------- .../codeql/reusables/threat-model-description.rst | 10 ++++++++++ 3 files changed, 14 insertions(+), 22 deletions(-) create mode 100644 docs/codeql/reusables/threat-model-description.rst diff --git a/docs/codeql/codeql-language-guides/customizing-library-models-for-csharp.rst b/docs/codeql/codeql-language-guides/customizing-library-models-for-csharp.rst index 09dcf36fc07..39b5ee30ee4 100644 --- a/docs/codeql/codeql-language-guides/customizing-library-models-for-csharp.rst +++ b/docs/codeql/codeql-language-guides/customizing-library-models-for-csharp.rst @@ -282,7 +282,7 @@ These are the same for both of the rows above as we are adding two summaries for - The second value ``Enumerable`` is the class (type) name. - The third value ``False`` is a flag that indicates whether or not the summary also applies to all overrides of the method. - The fourth value ``Select`` is the method name, along with the type parameters for the method. The names of the generic type parameters provided in the model must match the names of the generic type parameters in the method signature in the source code. -- The fifth value ``(System.Collections.Generic.IEnumerable,System.Func)`` is the method input type signature. The generics in the signature must match the generics in the method signature in the source code. +- The fifth value ``(System.Collections.Generic.IEnumerable,System.Func)`` is the method input type signature. The generics in the signature must match the generics in the method signature in the source code. The sixth value should be left empty and is out of scope for this documentation. The remaining values are used to define the ``access path``, the ``kind``, and the ``provenance`` (origin) of the summary definition. @@ -309,7 +309,7 @@ That is, the first row specifies that values can flow from the elements of the q Example: Add a ``neutral`` method ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This example shows how we can model a method as being neutral with respect to flow. We will also cover how to model a property by modeling the getter of the ``Now`` property of the ``DateTime`` class as neutral. +This example shows how we can model a method as being neutral with respect to flow. We will also cover how to model a property by modeling the getter of the ``Now`` property of the ``DateTime`` class as neutral. A neutral model is used to define that there is no flow through a method. .. code-block:: csharp @@ -346,13 +346,4 @@ The first four values identify the callable (in this case the getter of the ``No Threat models ------------- -.. include:: ../reusables/beta-note-threat-models.rst - -A threat model is a named class of dataflow sources that can be enabled or disabled independently. Threat models allow you to control the set of dataflow sources that you want to consider unsafe. For example, one codebase may only consider remote HTTP requests to be tainted, whereas another may also consider data from local files to be unsafe. You can use threat models to ensure that the relevant taint sources are used in a CodeQL analysis. - -The ``kind`` property of ``sourceModel`` determines which threat model a source is associated with. There are two main categories: - -- ``remote`` which represents requests and responses from the network. -- ``local`` which represents data from local files (``file``), command-line arguments (``commandargs``), database reads (``database``), and environment variables(``environment``). - -When running a CodeQL analysis, the ``remote`` threat model is included by default. You can optionally include other threat models as appropriate when using the CodeQL CLI and in GitHub code scanning. For more information, see `Analyzing your code with CodeQL queries `__ and `Customizing your advanced setup for code scanning `__. +.. include:: ../reusables/threat-model-description.rst diff --git a/docs/codeql/codeql-language-guides/customizing-library-models-for-java-and-kotlin.rst b/docs/codeql/codeql-language-guides/customizing-library-models-for-java-and-kotlin.rst index e8b69e20d81..dd7f352f6d0 100644 --- a/docs/codeql/codeql-language-guides/customizing-library-models-for-java-and-kotlin.rst +++ b/docs/codeql/codeql-language-guides/customizing-library-models-for-java-and-kotlin.rst @@ -297,13 +297,4 @@ The first four values identify the callable (in this case a method) to be modele Threat models ------------- -.. include:: ../reusables/beta-note-threat-models.rst - -A threat model is a named class of dataflow sources that can be enabled or disabled independently. Threat models allow you to control the set of dataflow sources that you want to consider unsafe. For example, one codebase may only consider remote HTTP requests to be tainted, whereas another may also consider data from local files to be unsafe. You can use threat models to ensure that the relevant taint sources are used in a CodeQL analysis. - -The ``kind`` property of the ``sourceModel`` determines which threat model a source is associated with. There are two main categories: - -- ``remote`` which represents requests and responses from the network. -- ``local`` which represents data from local files (``file``), command-line arguments (``commandargs``), database reads (``database``), and environment variables(``environment``). - -When running a CodeQL analysis, the ``remote`` threat model is included by default. You can optionally include other threat models as appropriate when using the CodeQL CLI and in GitHub code scanning. For more information, see `Analyzing your code with CodeQL queries `__ and `Customizing your advanced setup for code scanning `__. +.. include:: ../reusables/threat-model-description.rst diff --git a/docs/codeql/reusables/threat-model-description.rst b/docs/codeql/reusables/threat-model-description.rst new file mode 100644 index 00000000000..53a872487bf --- /dev/null +++ b/docs/codeql/reusables/threat-model-description.rst @@ -0,0 +1,10 @@ +.. include:: ../reusables/beta-note-threat-models.rst + +A threat model is a named class of dataflow sources that can be enabled or disabled independently. Threat models allow you to control the set of dataflow sources that you want to consider unsafe. For example, one codebase may only consider remote HTTP requests to be tainted, whereas another may also consider data from local files to be unsafe. You can use threat models to ensure that the relevant taint sources are used in a CodeQL analysis. + +The ``kind`` property of the ``sourceModel`` determines which threat model a source is associated with. There are two main categories: + +- ``remote`` which represents requests and responses from the network. +- ``local`` which represents data from local files (``file``), command-line arguments (``commandargs``), database reads (``database``), and environment variables(``environment``). + +When running a CodeQL analysis, the ``remote`` threat model is included by default. You can optionally include other threat models as appropriate when using the CodeQL CLI and in GitHub code scanning. For more information, see `Analyzing your code with CodeQL queries `__ and `Customizing your advanced setup for code scanning `__. From deb78b248b8d04aab1702c721d64db89de748a44 Mon Sep 17 00:00:00 2001 From: Max Schaefer <54907921+max-schaefer@users.noreply.github.com> Date: Tue, 9 Apr 2024 14:38:17 +0100 Subject: [PATCH 30/50] Apply suggestions from code review Co-authored-by: Taus --- .../test/AutomodelApplicationModeExtraction/Test.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java b/java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java index a8280bcaf27..9691cf86c15 100644 --- a/java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java +++ b/java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java @@ -103,10 +103,10 @@ class MoreTests { Files.delete( p // $ sinkModelCandidate=delete(Path):Argument[0] positiveSinkExample=delete(Path):Argument[0](path-injection) - ); // $ not a source candidate (return type is void) + ); // not a source candidate (return type is void) Files.deleteIfExists( p // $ sinkModelCandidate=deleteIfExists(Path):Argument[0] positiveSinkExample=deleteIfExists(Path):Argument[0](path-injection) - ); // $ not a source candidate (return type is boolean) + ); // not a source candidate (return type is boolean) } } \ No newline at end of file From 2a7420ce11ef59eca1d0228d972e534c3d3df8ec Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Tue, 9 Apr 2024 15:00:23 +0100 Subject: [PATCH 31/50] Update cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll Co-authored-by: Jeroen Ketema <93738568+jketema@users.noreply.github.com> --- cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll index 44f2e60daac..d8ce666a8cb 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll @@ -7,7 +7,7 @@ private module Input implements TypeFlowInput { // The address of a variable is always a single object i instanceof VariableAddressInstruction or - // A reference always points to a always a single object + // A reference always points to a single object i.getResultLanguageType().hasUnspecifiedType(any(ReferenceType rt), false) or // `this` is never an array From 291cc0a671a4bcb9d411f6b8016e61cb0209ad02 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Tue, 9 Apr 2024 15:25:13 +0100 Subject: [PATCH 32/50] C++: Anonymous namespaces provide internal linkage. --- cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll index d8ce666a8cb..ee493f00ce4 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll @@ -120,10 +120,13 @@ private module Input implements TypeFlowInput { * calls to this function. */ private predicate isPrivate(Function func) { + // static functions have internal linkage func.isStatic() or - func.getNamespace().getParentNamespace*().isInline() + // anonymous namespaces have internal linkage + func.getNamespace().getParentNamespace*().isAnonymous() or + // private member functions are only called internally from inside the class func.(MemberFunction).isPrivate() } From a53ef495ee937e58bfd401f47732a59887f1e06c Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Tue, 9 Apr 2024 16:04:20 +0100 Subject: [PATCH 33/50] C++: Simplify 'hasExactBufferType' and add comments. --- .../cpp/ir/dataflow/internal/TypeFlow.qll | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll index ee493f00ce4..69f94dad91b 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll @@ -3,6 +3,29 @@ private import semmle.code.cpp.ir.IR private import codeql.typeflow.TypeFlow private module Input implements TypeFlowInput { + /** Holds if `alloc` dynamically allocates a single object. */ + private predicate isSingleObjectAllocation(AllocationExpr alloc) { + // i.e., `new int`; + alloc instanceof NewExpr + or + // i.e., `malloc(sizeof(int))` + exists(SizeofTypeOperator sizeOf | sizeOf = alloc.getSizeExpr() | + not sizeOf.getTypeOperand().getUnspecifiedType() instanceof ArrayType + ) + } + + /** + * Holds if `i` is the result of a dynamic allocation. + * + * `isObject` is `true` if the allocation allocated a single object, + * and `false` otherwise. + */ + private predicate isAllocation(Instruction i, boolean isObject) { + exists(AllocationExpr alloc | alloc = i.getUnconvertedResultExpression() | + if isSingleObjectAllocation(alloc) then isObject = true else isObject = false + ) + } + private predicate hasExactSingleType(Instruction i) { // The address of a variable is always a single object i instanceof VariableAddressInstruction @@ -14,23 +37,16 @@ private module Input implements TypeFlowInput { i instanceof InitializeThisInstruction or // An allocation of a non-array object - exists(AllocationExpr alloc | alloc = i.getUnconvertedResultExpression() | - // i.e., `new int`; - alloc instanceof NewExpr - or - // i.e., `malloc(sizeof(int))` - exists(SizeofTypeOperator sizeOf | sizeOf = alloc.getSizeExpr() | - not sizeOf.getTypeOperand().getUnspecifiedType() instanceof ArrayType - ) - ) + isAllocation(i, true) } private predicate hasExactBufferType(Instruction i) { // Anything with an array type is a buffer i.getResultLanguageType().hasUnspecifiedType(any(ArrayType at), false) or - not hasExactSingleType(i) and - i.getUnconvertedResultExpression() instanceof AllocationExpr + // An allocation expression that we couldn't conclude allocated a single + // expression is assigned a buffer type. + isAllocation(i, false) } private newtype TTypeFlowNode = From b9cfeaf6140617109572152b84ab84fbf4be067e Mon Sep 17 00:00:00 2001 From: Dave Bartolomeo Date: Tue, 9 Apr 2024 12:41:32 -0400 Subject: [PATCH 34/50] Add test case --- java/ql/test/query-tests/SpuriousJavadocParam/Test.java | 1 + java/ql/test/query-tests/SpuriousJavadocParam/test.expected | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/java/ql/test/query-tests/SpuriousJavadocParam/Test.java b/java/ql/test/query-tests/SpuriousJavadocParam/Test.java index 434353f4c67..d8891afb756 100644 --- a/java/ql/test/query-tests/SpuriousJavadocParam/Test.java +++ b/java/ql/test/query-tests/SpuriousJavadocParam/Test.java @@ -128,6 +128,7 @@ public class Test { /** * @param exists + * @param does not * @param i exists * @param k does not */ diff --git a/java/ql/test/query-tests/SpuriousJavadocParam/test.expected b/java/ql/test/query-tests/SpuriousJavadocParam/test.expected index 0ade1a7d519..f184473ea4a 100644 --- a/java/ql/test/query-tests/SpuriousJavadocParam/test.expected +++ b/java/ql/test/query-tests/SpuriousJavadocParam/test.expected @@ -13,4 +13,5 @@ | Test.java:118:6:118:12 | @param | @param tag "T" does not match any actual type parameter of type "GenericInterface". | | Test.java:119:6:119:12 | @param | @param tag "" does not match any actual type parameter of type "GenericInterface". | | Test.java:125:6:125:12 | @param | @param tag "k" does not match any actual type parameter or record parameter of record "SomeRecord". | -| Test.java:132:6:132:12 | @param | @param tag "k" does not match any actual type parameter or record parameter of record "GenericRecord". | +| Test.java:131:6:131:12 | @param | @param tag "" does not match any actual type parameter or record parameter of record "GenericRecord". | +| Test.java:133:6:133:12 | @param | @param tag "k" does not match any actual type parameter or record parameter of record "GenericRecord". | From 4fed3cf12d9145b2d891009ff409853da9a61ebb Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 10 Apr 2024 11:31:34 +0200 Subject: [PATCH 35/50] Python: Fix `RemoteFlowSourceFromCsv` --- python/ql/lib/semmle/python/frameworks/data/ModelsAsData.qll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ql/lib/semmle/python/frameworks/data/ModelsAsData.qll b/python/ql/lib/semmle/python/frameworks/data/ModelsAsData.qll index 34e48439271..2f4c74ea9e4 100644 --- a/python/ql/lib/semmle/python/frameworks/data/ModelsAsData.qll +++ b/python/ql/lib/semmle/python/frameworks/data/ModelsAsData.qll @@ -22,7 +22,7 @@ private import semmle.python.dataflow.new.FlowSummary /** * A remote flow source originating from a CSV source row. */ -private class RemoteFlowSourceFromCsv extends RemoteFlowSource { +private class RemoteFlowSourceFromCsv extends RemoteFlowSource::Range { RemoteFlowSourceFromCsv() { this = ModelOutput::getASourceNode("remote").asSource() } override string getSourceType() { result = "Remote flow (from model)" } From b4829addf7b3c2ac0121e6dc0222d87381a166cc Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Wed, 10 Apr 2024 10:48:23 +0100 Subject: [PATCH 36/50] Add comments with link to online doc --- go/extractor/extractor.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/go/extractor/extractor.go b/go/extractor/extractor.go index 1a60c2bf982..acfccf33e3c 100644 --- a/go/extractor/extractor.go +++ b/go/extractor/extractor.go @@ -388,6 +388,8 @@ func extractUniverseScope() { } // extractObjects extracts all objects declared in the given scope +// For more information on objects, see: +// https://github.com/golang/example/blob/master/gotypes/README.md#objects func extractObjects(tw *trap.Writer, scope *types.Scope, scopeLabel trap.Label) { for _, name := range scope.Names() { obj := scope.Lookup(name) @@ -440,6 +442,8 @@ func extractMethod(tw *trap.Writer, meth *types.Func) trap.Label { } // extractObject extracts a single object and emits it to the objects table. +// For more information on objects, see: +// https://github.com/golang/example/blob/master/gotypes/README.md#objects func extractObject(tw *trap.Writer, obj types.Object, lbl trap.Label) { name := obj.Name() isBuiltin := obj.Parent() == types.Universe @@ -487,6 +491,8 @@ func extractObject(tw *trap.Writer, obj types.Object, lbl trap.Label) { } // extractObjectTypes extracts type and receiver information for all objects +// For more information on objects, see: +// https://github.com/golang/example/blob/master/gotypes/README.md#objects func extractObjectTypes(tw *trap.Writer) { // calling `extractType` on a named type will extract all methods defined // on it, which will add new objects. Therefore we need to do this first @@ -497,11 +503,13 @@ func extractObjectTypes(tw *trap.Writer) { } changed = tw.ForEachObject(emitObjectType) if changed { - log.Printf("Warning: more objects were labeled while emitted object types") + log.Printf("Warning: more objects were labeled while emitting object types") } } // extractObjectType extracts type and receiver information for a given object +// For more information on objects, see: +// https://github.com/golang/example/blob/master/gotypes/README.md#objects func extractObjectType(tw *trap.Writer, obj types.Object, lbl trap.Label) { if tp := obj.Type(); tp != nil { extractType(tw, tp) From bee54e42479867a901e546d4782f3f80f73a1db8 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Fri, 5 Apr 2024 14:41:10 +0200 Subject: [PATCH 37/50] C#: Add desktop runtime dependencies integration test. --- .../Assemblies.expected | 210 ++++++++++++++++++ .../Assemblies.ql | 21 ++ .../Program.cs | 6 + .../global.json | 5 + .../test.py | 3 + 5 files changed, 245 insertions(+) create mode 100644 csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.expected create mode 100644 csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.ql create mode 100644 csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Program.cs create mode 100644 csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/global.json create mode 100644 csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/test.py diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.expected b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.expected new file mode 100644 index 00000000000..f4f8290eb75 --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.expected @@ -0,0 +1,210 @@ +| [...]/Humanizer.dll | +| [...]/Microsoft.Bcl.AsyncInterfaces.dll | +| [...]/Microsoft.Build.Framework.dll | +| [...]/Microsoft.Build.dll | +| [...]/Microsoft.CSharp.dll | +| [...]/Microsoft.CodeAnalysis.CSharp.Workspaces.dll | +| [...]/Microsoft.CodeAnalysis.CSharp.dll | +| [...]/Microsoft.CodeAnalysis.VisualBasic.Workspaces.dll | +| [...]/Microsoft.CodeAnalysis.VisualBasic.dll | +| [...]/Microsoft.CodeAnalysis.Workspaces.dll | +| [...]/Microsoft.CodeAnalysis.dll | +| [...]/Microsoft.NET.StringTools.dll | +| [...]/Microsoft.VisualBasic.Core.dll | +| [...]/Microsoft.VisualBasic.dll | +| [...]/Microsoft.Win32.Primitives.dll | +| [...]/Microsoft.Win32.Registry.dll | +| [...]/Microsoft.Win32.SystemEvents.dll | +| [...]/Mono.Posix.NETStandard.dll | +| [...]/Newtonsoft.Json.dll | +| [...]/Semmle.Autobuild.CSharp.dll | +| [...]/Semmle.Autobuild.Shared.dll | +| [...]/Semmle.Extraction.CSharp.DependencyFetching.dll | +| [...]/Semmle.Extraction.CSharp.Driver.dll | +| [...]/Semmle.Extraction.CSharp.Standalone.dll | +| [...]/Semmle.Extraction.CSharp.Util.dll | +| [...]/Semmle.Extraction.CSharp.dll | +| [...]/Semmle.Extraction.dll | +| [...]/Semmle.Util.dll | +| [...]/System.AppContext.dll | +| [...]/System.Buffers.dll | +| [...]/System.Collections.Concurrent.dll | +| [...]/System.Collections.Immutable.dll | +| [...]/System.Collections.NonGeneric.dll | +| [...]/System.Collections.Specialized.dll | +| [...]/System.Collections.dll | +| [...]/System.ComponentModel.Annotations.dll | +| [...]/System.ComponentModel.DataAnnotations.dll | +| [...]/System.ComponentModel.EventBasedAsync.dll | +| [...]/System.ComponentModel.Primitives.dll | +| [...]/System.ComponentModel.TypeConverter.dll | +| [...]/System.ComponentModel.dll | +| [...]/System.Composition.AttributedModel.dll | +| [...]/System.Composition.Convention.dll | +| [...]/System.Composition.Hosting.dll | +| [...]/System.Composition.Runtime.dll | +| [...]/System.Composition.TypedParts.dll | +| [...]/System.Configuration.ConfigurationManager.dll | +| [...]/System.Configuration.dll | +| [...]/System.Console.dll | +| [...]/System.Core.dll | +| [...]/System.Data.Common.dll | +| [...]/System.Data.DataSetExtensions.dll | +| [...]/System.Data.dll | +| [...]/System.Diagnostics.Contracts.dll | +| [...]/System.Diagnostics.Debug.dll | +| [...]/System.Diagnostics.DiagnosticSource.dll | +| [...]/System.Diagnostics.EventLog.dll | +| [...]/System.Diagnostics.FileVersionInfo.dll | +| [...]/System.Diagnostics.Process.dll | +| [...]/System.Diagnostics.StackTrace.dll | +| [...]/System.Diagnostics.TextWriterTraceListener.dll | +| [...]/System.Diagnostics.Tools.dll | +| [...]/System.Diagnostics.TraceSource.dll | +| [...]/System.Diagnostics.Tracing.dll | +| [...]/System.Drawing.Common.dll | +| [...]/System.Drawing.Primitives.dll | +| [...]/System.Drawing.dll | +| [...]/System.Dynamic.Runtime.dll | +| [...]/System.Formats.Asn1.dll | +| [...]/System.Formats.Tar.dll | +| [...]/System.Globalization.Calendars.dll | +| [...]/System.Globalization.Extensions.dll | +| [...]/System.Globalization.dll | +| [...]/System.IO.Compression.Brotli.dll | +| [...]/System.IO.Compression.FileSystem.dll | +| [...]/System.IO.Compression.ZipFile.dll | +| [...]/System.IO.Compression.dll | +| [...]/System.IO.FileSystem.AccessControl.dll | +| [...]/System.IO.FileSystem.DriveInfo.dll | +| [...]/System.IO.FileSystem.Primitives.dll | +| [...]/System.IO.FileSystem.Watcher.dll | +| [...]/System.IO.FileSystem.dll | +| [...]/System.IO.IsolatedStorage.dll | +| [...]/System.IO.MemoryMappedFiles.dll | +| [...]/System.IO.Pipelines.dll | +| [...]/System.IO.Pipes.AccessControl.dll | +| [...]/System.IO.Pipes.dll | +| [...]/System.IO.UnmanagedMemoryStream.dll | +| [...]/System.IO.dll | +| [...]/System.Linq.Expressions.dll | +| [...]/System.Linq.Parallel.dll | +| [...]/System.Linq.Queryable.dll | +| [...]/System.Linq.dll | +| [...]/System.Memory.dll | +| [...]/System.Net.Http.Json.dll | +| [...]/System.Net.Http.dll | +| [...]/System.Net.HttpListener.dll | +| [...]/System.Net.Mail.dll | +| [...]/System.Net.NameResolution.dll | +| [...]/System.Net.NetworkInformation.dll | +| [...]/System.Net.Ping.dll | +| [...]/System.Net.Primitives.dll | +| [...]/System.Net.Quic.dll | +| [...]/System.Net.Requests.dll | +| [...]/System.Net.Security.dll | +| [...]/System.Net.ServicePoint.dll | +| [...]/System.Net.Sockets.dll | +| [...]/System.Net.WebClient.dll | +| [...]/System.Net.WebHeaderCollection.dll | +| [...]/System.Net.WebProxy.dll | +| [...]/System.Net.WebSockets.Client.dll | +| [...]/System.Net.WebSockets.dll | +| [...]/System.Net.dll | +| [...]/System.Numerics.Vectors.dll | +| [...]/System.Numerics.dll | +| [...]/System.ObjectModel.dll | +| [...]/System.Private.CoreLib.dll | +| [...]/System.Private.DataContractSerialization.dll | +| [...]/System.Private.Uri.dll | +| [...]/System.Private.Xml.Linq.dll | +| [...]/System.Private.Xml.dll | +| [...]/System.Reflection.DispatchProxy.dll | +| [...]/System.Reflection.Emit.ILGeneration.dll | +| [...]/System.Reflection.Emit.Lightweight.dll | +| [...]/System.Reflection.Emit.dll | +| [...]/System.Reflection.Extensions.dll | +| [...]/System.Reflection.Metadata.dll | +| [...]/System.Reflection.MetadataLoadContext.dll | +| [...]/System.Reflection.Primitives.dll | +| [...]/System.Reflection.TypeExtensions.dll | +| [...]/System.Reflection.dll | +| [...]/System.Resources.Reader.dll | +| [...]/System.Resources.ResourceManager.dll | +| [...]/System.Resources.Writer.dll | +| [...]/System.Runtime.CompilerServices.Unsafe.dll | +| [...]/System.Runtime.CompilerServices.VisualC.dll | +| [...]/System.Runtime.Extensions.dll | +| [...]/System.Runtime.Handles.dll | +| [...]/System.Runtime.InteropServices.JavaScript.dll | +| [...]/System.Runtime.InteropServices.RuntimeInformation.dll | +| [...]/System.Runtime.InteropServices.dll | +| [...]/System.Runtime.Intrinsics.dll | +| [...]/System.Runtime.Loader.dll | +| [...]/System.Runtime.Numerics.dll | +| [...]/System.Runtime.Serialization.Formatters.dll | +| [...]/System.Runtime.Serialization.Json.dll | +| [...]/System.Runtime.Serialization.Primitives.dll | +| [...]/System.Runtime.Serialization.Xml.dll | +| [...]/System.Runtime.Serialization.dll | +| [...]/System.Runtime.dll | +| [...]/System.Security.AccessControl.dll | +| [...]/System.Security.Claims.dll | +| [...]/System.Security.Cryptography.Algorithms.dll | +| [...]/System.Security.Cryptography.Cng.dll | +| [...]/System.Security.Cryptography.Csp.dll | +| [...]/System.Security.Cryptography.Encoding.dll | +| [...]/System.Security.Cryptography.OpenSsl.dll | +| [...]/System.Security.Cryptography.Primitives.dll | +| [...]/System.Security.Cryptography.ProtectedData.dll | +| [...]/System.Security.Cryptography.X509Certificates.dll | +| [...]/System.Security.Cryptography.dll | +| [...]/System.Security.Permissions.dll | +| [...]/System.Security.Principal.Windows.dll | +| [...]/System.Security.Principal.dll | +| [...]/System.Security.SecureString.dll | +| [...]/System.Security.dll | +| [...]/System.ServiceModel.Web.dll | +| [...]/System.ServiceProcess.dll | +| [...]/System.Text.Encoding.CodePages.dll | +| [...]/System.Text.Encoding.Extensions.dll | +| [...]/System.Text.Encoding.dll | +| [...]/System.Text.Encodings.Web.dll | +| [...]/System.Text.Json.dll | +| [...]/System.Text.RegularExpressions.dll | +| [...]/System.Threading.Channels.dll | +| [...]/System.Threading.Overlapped.dll | +| [...]/System.Threading.Tasks.Dataflow.dll | +| [...]/System.Threading.Tasks.Extensions.dll | +| [...]/System.Threading.Tasks.Parallel.dll | +| [...]/System.Threading.Tasks.dll | +| [...]/System.Threading.Thread.dll | +| [...]/System.Threading.ThreadPool.dll | +| [...]/System.Threading.Timer.dll | +| [...]/System.Threading.dll | +| [...]/System.Transactions.Local.dll | +| [...]/System.Transactions.dll | +| [...]/System.ValueTuple.dll | +| [...]/System.Web.HttpUtility.dll | +| [...]/System.Web.dll | +| [...]/System.Windows.Extensions.dll | +| [...]/System.Windows.dll | +| [...]/System.Xml.Linq.dll | +| [...]/System.Xml.ReaderWriter.dll | +| [...]/System.Xml.Serialization.dll | +| [...]/System.Xml.XDocument.dll | +| [...]/System.Xml.XPath.XDocument.dll | +| [...]/System.Xml.XPath.dll | +| [...]/System.Xml.XmlDocument.dll | +| [...]/System.Xml.XmlSerializer.dll | +| [...]/System.Xml.dll | +| [...]/System.dll | +| [...]/WindowsBase.dll | +| [...]/mscorlib.dll | +| [...]/netstandard.dll | +| [...]/zh-Hant/Microsoft.CodeAnalysis.CSharp.Workspaces.resources.dll | +| [...]/zh-Hant/Microsoft.CodeAnalysis.CSharp.resources.dll | +| [...]/zh-Hant/Microsoft.CodeAnalysis.VisualBasic.Workspaces.resources.dll | +| [...]/zh-Hant/Microsoft.CodeAnalysis.VisualBasic.resources.dll | +| [...]/zh-Hant/Microsoft.CodeAnalysis.Workspaces.resources.dll | +| [...]/zh-Hant/Microsoft.CodeAnalysis.resources.dll | diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.ql b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.ql new file mode 100644 index 00000000000..3bb648c579a --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.ql @@ -0,0 +1,21 @@ +import csharp + +private string getPath(Assembly a) { + not a.getCompilation().getOutputAssembly() = a and + exists(string s | s = a.getFile().getAbsolutePath() | + result = + "[...]" + + s.substring(s.indexOf("test-db/working/") + "test-db/working/".length() + 16 + + "/packages".length(), s.length()) + or + exists(string sub | sub = "csharp/tools/" + ["osx64", "linux64"] | + result = "[...]" + s.substring(s.indexOf(sub) + sub.length(), s.length()) + ) + or + result = s and + not exists(s.indexOf(["test-db/working/", "csharp/tools/"])) + ) +} + +from Assembly a +select getPath(a) diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Program.cs b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Program.cs new file mode 100644 index 00000000000..39a9e95bb6e --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Program.cs @@ -0,0 +1,6 @@ +class Program +{ + static void Main(string[] args) + { + } +} \ No newline at end of file diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/global.json b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/global.json new file mode 100644 index 00000000000..d54915e8d4d --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/global.json @@ -0,0 +1,5 @@ +{ + "sdk": { + "version": "8.0.101" + } +} diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/test.py b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/test.py new file mode 100644 index 00000000000..a17966e148a --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/test.py @@ -0,0 +1,3 @@ +from create_database_utils import * + +run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"]) From 6299d9cecd1e562c9859a2ee8b0f3580ed391ff1 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Wed, 20 Mar 2024 15:19:13 +0100 Subject: [PATCH 38/50] C#: Introduce AssemblyPath and re-factor AssemblyCache to use this instead of strings. --- .../AssemblyCache.cs | 108 +++++++++++++----- .../DependencyManager.Nuget.cs | 8 +- .../DependencyManager.cs | 22 ++-- 3 files changed, 92 insertions(+), 46 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs index 3d6672560b6..0d2393faa0a 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs @@ -1,10 +1,84 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.IO; using System.Linq; using Semmle.Util.Logging; namespace Semmle.Extraction.CSharp.DependencyFetching { + /// + /// Used to represent a path to an assembly or a directory containing assemblies + /// and a selector function to determine which files to include, when indexing the assemblies. + /// + internal sealed class AssemblyPath(string p, Func includeFileName) + { + public string Path => p; + + public AssemblyPath(string p) : this(p, _ => true) { } + + public static implicit operator AssemblyPath(string path) => new(path); + + /// + /// Finds all assemblies nested within the directory `path` + /// and adds them to the a list of assembly names to index. + /// Indexing is performed at a later stage. This only collects the names. + /// + /// The directory to index. + private void AddReferenceDirectory(List dllsToIndex, ILogger logger) + { + foreach (var dll in new DirectoryInfo(p).EnumerateFiles("*.dll", SearchOption.AllDirectories)) + { + if (includeFileName(dll.Name)) + { + dllsToIndex.Add(dll.FullName); + } + else + { + logger.LogInfo($"AssemblyPath: Skipping {dll.FullName}."); + } + } + } + + /// + /// Finds all assemblies in `p` that should be indexed and adds them to + /// the list of assembly names to index. + /// + /// List of assembly names to index. + /// Logger + public void Process(List dllsToIndex, ILogger logger) + { + if (File.Exists(p)) + { + if (includeFileName(System.IO.Path.GetFileName(p))) + { + dllsToIndex.Add(p); + } + else + { + logger.LogInfo($"AssemblyPath: Skipping {p}."); + } + return; + } + + if (Directory.Exists(p)) + { + logger.LogInfo($"AssemblyPath: Finding reference DLLs in {p}..."); + AddReferenceDirectory(dllsToIndex, logger); + } + else + { + logger.LogInfo("AssemblyCache: Path not found: " + p); + } + } + + public override bool Equals(object? obj) => + obj is AssemblyPath ap && p.Equals(ap.Path); + + public override int GetHashCode() => p.GetHashCode(); + + public override string ToString() => p; + } + /// /// Manages the set of assemblies. /// Searches for assembly DLLs, indexes them and provides @@ -20,44 +94,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// assembly cache. /// /// Callback for progress. - public AssemblyCache(IEnumerable paths, IEnumerable frameworkPaths, ILogger logger) + public AssemblyCache(IEnumerable paths, IEnumerable frameworkPaths, ILogger logger) { this.logger = logger; foreach (var path in paths) { - if (File.Exists(path)) - { - dllsToIndex.Add(path); - continue; - } - - if (Directory.Exists(path)) - { - logger.LogInfo($"Finding reference DLLs in {path}..."); - AddReferenceDirectory(path); - } - else - { - logger.LogInfo("AssemblyCache: Path not found: " + path); - } + path.Process(dllsToIndex, logger); } IndexReferences(frameworkPaths); } - /// - /// Finds all assemblies nested within a directory - /// and adds them to its index. - /// (Indexing is performed at a later stage by IndexReferences()). - /// - /// The directory to index. - private void AddReferenceDirectory(string dir) - { - foreach (var dll in new DirectoryInfo(dir).EnumerateFiles("*.dll", SearchOption.AllDirectories)) - { - dllsToIndex.Add(dll.FullName); - } - } - /// /// Indexes all DLLs we have located. /// Because this is a potentially time-consuming operation, it is put into a separate stage. diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs index 1f3f44425d5..e808235e54d 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs @@ -12,7 +12,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { public sealed partial class DependencyManager { - private void RestoreNugetPackages(List allNonBinaryFiles, IEnumerable allProjects, IEnumerable allSolutions, HashSet dllPaths) + private void RestoreNugetPackages(List allNonBinaryFiles, IEnumerable allProjects, IEnumerable allSolutions, HashSet dllPaths) { try { @@ -55,7 +55,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } nugetPackageDllPaths.ExceptWith(excludedPaths); - dllPaths.UnionWith(nugetPackageDllPaths); + dllPaths.UnionWith(nugetPackageDllPaths.Select(p => new AssemblyPath(p))); } catch (Exception exc) { @@ -72,7 +72,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching .Paths .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d)) .ToList(); - dllPaths.UnionWith(paths); + dllPaths.UnionWith(paths.Select(p => new AssemblyPath(p))); LogAllUnusedPackages(dependencies); DownloadMissingPackages(allNonBinaryFiles, dllPaths); @@ -148,7 +148,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString())); } - private void DownloadMissingPackages(List allFiles, ISet dllPaths, bool withNugetConfig = true) + private void DownloadMissingPackages(List allFiles, ISet dllPaths, bool withNugetConfig = true) { var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo); var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index 8d63d7f42b2..55c41f31601 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -46,7 +46,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// /// Performs C# dependency fetching. /// - /// Dependency fetching options + /// Path to directory containing source code. /// Logger for dependency fetching progress. public DependencyManager(string srcDir, ILogger logger) { @@ -91,7 +91,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching this.generatedSources = new(); var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj").ToList(); var allSolutions = allNonBinaryFiles.SelectFileNamesByExtension(".sln").ToList(); - var dllPaths = allFiles.SelectFileNamesByExtension(".dll").ToHashSet(); + var dllPaths = allFiles.SelectFileNamesByExtension(".dll").Select(x => x).ToHashSet(); logger.LogInfo($"Found {allFiles.Count} files, {nonGeneratedSources.Count} source files, {allProjects.Count} project files, {allSolutions.Count} solution files, {dllPaths.Count} DLLs."); @@ -192,7 +192,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching ]); } - private HashSet AddFrameworkDlls(HashSet dllPaths) + private HashSet AddFrameworkDlls(HashSet dllPaths) { var frameworkLocations = new HashSet(); @@ -230,7 +230,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching continue; } - dllPaths.UnionWith(dlls); + dllPaths.UnionWith(dlls.Select(x => x)); frameworkLocations.UnionWith(dlls); } catch (Exception e) @@ -284,7 +284,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet dllPaths, ISet frameworkLocations) + private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet dllPaths, ISet frameworkLocations) { var versionFolders = GetPackageVersionSubDirectories(frameworkPath); if (versionFolders.Length > 1) @@ -313,7 +313,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching .ToArray(); } - private void RemoveFrameworkNugetPackages(ISet dllPaths, int fromIndex = 0) + private void RemoveFrameworkNugetPackages(ISet dllPaths, int fromIndex = 0) { var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks; for (var i = fromIndex; i < packagesInPrioOrder.Length; i++) @@ -322,7 +322,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - private void AddNetFrameworkDlls(ISet dllPaths, ISet frameworkLocations) + private void AddNetFrameworkDlls(ISet dllPaths, ISet frameworkLocations) { // Multiple dotnet framework packages could be present. // The order of the packages is important, we're adding the first one that is present in the nuget cache. @@ -375,7 +375,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching frameworkLocations.Add(runtimeLocation); } - private void RemoveNugetPackageReference(string packagePrefix, ISet dllPaths) + private void RemoveNugetPackageReference(string packagePrefix, ISet dllPaths) { var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant(); if (packageFolder == null) @@ -384,7 +384,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } var packagePathPrefix = Path.Combine(packageFolder, packagePrefix.ToLowerInvariant()); - var toRemove = dllPaths.Where(s => s.StartsWith(packagePathPrefix, StringComparison.InvariantCultureIgnoreCase)); + var toRemove = dllPaths.Where(s => s.Path.StartsWith(packagePathPrefix, StringComparison.InvariantCultureIgnoreCase)); foreach (var path in toRemove) { dllPaths.Remove(path); @@ -397,7 +397,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return fileContent.IsNewProjectStructureUsed && fileContent.UseAspNetCoreDlls; } - private void AddAspNetCoreFrameworkDlls(ISet dllPaths, ISet frameworkLocations) + private void AddAspNetCoreFrameworkDlls(ISet dllPaths, ISet frameworkLocations) { if (!IsAspNetCoreDetected()) { @@ -419,7 +419,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - private void AddMicrosoftWindowsDesktopDlls(ISet dllPaths, ISet frameworkLocations) + private void AddMicrosoftWindowsDesktopDlls(ISet dllPaths, ISet frameworkLocations) { if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework, packageDirectory) is string windowsDesktopApp) { From 646b272b4e0befe19902a10c3dee5af2e6870836 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Fri, 5 Apr 2024 10:01:20 +0200 Subject: [PATCH 39/50] C#: Move the AssemblyPath class to its own file. --- .../AssemblyCache.cs | 77 +----------------- .../AssemblyPath.cs | 80 +++++++++++++++++++ 2 files changed, 81 insertions(+), 76 deletions(-) create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyPath.cs diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs index 0d2393faa0a..f1ee65319ab 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs @@ -1,84 +1,9 @@ -using System; -using System.Collections.Generic; -using System.IO; +using System.Collections.Generic; using System.Linq; using Semmle.Util.Logging; namespace Semmle.Extraction.CSharp.DependencyFetching { - /// - /// Used to represent a path to an assembly or a directory containing assemblies - /// and a selector function to determine which files to include, when indexing the assemblies. - /// - internal sealed class AssemblyPath(string p, Func includeFileName) - { - public string Path => p; - - public AssemblyPath(string p) : this(p, _ => true) { } - - public static implicit operator AssemblyPath(string path) => new(path); - - /// - /// Finds all assemblies nested within the directory `path` - /// and adds them to the a list of assembly names to index. - /// Indexing is performed at a later stage. This only collects the names. - /// - /// The directory to index. - private void AddReferenceDirectory(List dllsToIndex, ILogger logger) - { - foreach (var dll in new DirectoryInfo(p).EnumerateFiles("*.dll", SearchOption.AllDirectories)) - { - if (includeFileName(dll.Name)) - { - dllsToIndex.Add(dll.FullName); - } - else - { - logger.LogInfo($"AssemblyPath: Skipping {dll.FullName}."); - } - } - } - - /// - /// Finds all assemblies in `p` that should be indexed and adds them to - /// the list of assembly names to index. - /// - /// List of assembly names to index. - /// Logger - public void Process(List dllsToIndex, ILogger logger) - { - if (File.Exists(p)) - { - if (includeFileName(System.IO.Path.GetFileName(p))) - { - dllsToIndex.Add(p); - } - else - { - logger.LogInfo($"AssemblyPath: Skipping {p}."); - } - return; - } - - if (Directory.Exists(p)) - { - logger.LogInfo($"AssemblyPath: Finding reference DLLs in {p}..."); - AddReferenceDirectory(dllsToIndex, logger); - } - else - { - logger.LogInfo("AssemblyCache: Path not found: " + p); - } - } - - public override bool Equals(object? obj) => - obj is AssemblyPath ap && p.Equals(ap.Path); - - public override int GetHashCode() => p.GetHashCode(); - - public override string ToString() => p; - } - /// /// Manages the set of assemblies. /// Searches for assembly DLLs, indexes them and provides diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyPath.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyPath.cs new file mode 100644 index 00000000000..816e0fef6a7 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyPath.cs @@ -0,0 +1,80 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Semmle.Util.Logging; + +namespace Semmle.Extraction.CSharp.DependencyFetching +{ + /// + /// Used to represent a path to an assembly or a directory containing assemblies + /// and a selector function to determine which files to include, when indexing the assemblies. + /// + internal sealed class AssemblyPath(string p, Func includeFileName) + { + public string Path => p; + + public AssemblyPath(string p) : this(p, _ => true) { } + + public static implicit operator AssemblyPath(string path) => new(path); + + /// + /// Finds all assemblies nested within the directory `path` + /// and adds them to the a list of assembly names to index. + /// Indexing is performed at a later stage. This only collects the names. + /// + /// The directory to index. + private void AddReferenceDirectory(List dllsToIndex, ILogger logger) + { + foreach (var dll in new DirectoryInfo(p).EnumerateFiles("*.dll", SearchOption.AllDirectories)) + { + if (includeFileName(dll.Name)) + { + dllsToIndex.Add(dll.FullName); + } + else + { + logger.LogInfo($"AssemblyPath: Skipping {dll.FullName}."); + } + } + } + + /// + /// Finds all assemblies in `p` that should be indexed and adds them to + /// the list of assembly names to index. + /// + /// List of assembly names to index. + /// Logger + public void Process(List dllsToIndex, ILogger logger) + { + if (File.Exists(p)) + { + if (includeFileName(System.IO.Path.GetFileName(p))) + { + dllsToIndex.Add(p); + } + else + { + logger.LogInfo($"AssemblyPath: Skipping {p}."); + } + return; + } + + if (Directory.Exists(p)) + { + logger.LogInfo($"AssemblyPath: Finding reference DLLs in {p}..."); + AddReferenceDirectory(dllsToIndex, logger); + } + else + { + logger.LogInfo("AssemblyCache: Path not found: " + p); + } + } + + public override bool Equals(object? obj) => + obj is AssemblyPath ap && p.Equals(ap.Path); + + public override int GetHashCode() => p.GetHashCode(); + + public override string ToString() => p; + } +} From d04bf6b6d637055103b795c00bbd9872c7276c96 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Fri, 5 Apr 2024 14:55:55 +0200 Subject: [PATCH 40/50] C#: Don't include Semmle.* dlls, if the executing runtime is used as framework. --- .../DependencyManager.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index 55c41f31601..228f26850f8 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -368,10 +368,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - runtimeLocation ??= Runtime.ExecutingRuntime; + if (runtimeLocation is null) + { + runtimeLocation ??= Runtime.ExecutingRuntime; + dllPaths.Add(new AssemblyPath(runtimeLocation, name => !name.StartsWith("Semmle."))); + } + else + { + dllPaths.Add(runtimeLocation); + } logger.LogInfo($".NET runtime location selected: {runtimeLocation}"); - dllPaths.Add(runtimeLocation); frameworkLocations.Add(runtimeLocation); } From 2bea927d43d2afbc5d5945abd1161e272992774d Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Fri, 5 Apr 2024 14:56:20 +0200 Subject: [PATCH 41/50] C#: Update expected test output. --- .../Assemblies.expected | 9 --------- 1 file changed, 9 deletions(-) diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.expected b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.expected index f4f8290eb75..772f640789e 100644 --- a/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.expected +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.expected @@ -17,15 +17,6 @@ | [...]/Microsoft.Win32.SystemEvents.dll | | [...]/Mono.Posix.NETStandard.dll | | [...]/Newtonsoft.Json.dll | -| [...]/Semmle.Autobuild.CSharp.dll | -| [...]/Semmle.Autobuild.Shared.dll | -| [...]/Semmle.Extraction.CSharp.DependencyFetching.dll | -| [...]/Semmle.Extraction.CSharp.Driver.dll | -| [...]/Semmle.Extraction.CSharp.Standalone.dll | -| [...]/Semmle.Extraction.CSharp.Util.dll | -| [...]/Semmle.Extraction.CSharp.dll | -| [...]/Semmle.Extraction.dll | -| [...]/Semmle.Util.dll | | [...]/System.AppContext.dll | | [...]/System.Buffers.dll | | [...]/System.Collections.Concurrent.dll | From 9eb13833faeac9c2bfdb2eaf247f1ee61907d7ff Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Tue, 9 Apr 2024 11:57:25 +0200 Subject: [PATCH 42/50] C#: Code quality improvements. --- .../AssemblyCache.cs | 4 +-- ...emblyPath.cs => AssemblyLookupLocation.cs} | 26 +++++++++---------- .../DependencyManager.Nuget.cs | 8 +++--- .../DependencyManager.cs | 20 +++++++------- 4 files changed, 29 insertions(+), 29 deletions(-) rename csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/{AssemblyPath.cs => AssemblyLookupLocation.cs} (67%) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs index f1ee65319ab..62a053870e9 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs @@ -19,12 +19,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// assembly cache. /// /// Callback for progress. - public AssemblyCache(IEnumerable paths, IEnumerable frameworkPaths, ILogger logger) + public AssemblyCache(IEnumerable paths, IEnumerable frameworkPaths, ILogger logger) { this.logger = logger; foreach (var path in paths) { - path.Process(dllsToIndex, logger); + dllsToIndex.AddRange(path.GetDlls(logger)); } IndexReferences(frameworkPaths); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyPath.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs similarity index 67% rename from csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyPath.cs rename to csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs index 816e0fef6a7..380c60cab56 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyPath.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs @@ -9,13 +9,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// Used to represent a path to an assembly or a directory containing assemblies /// and a selector function to determine which files to include, when indexing the assemblies. /// - internal sealed class AssemblyPath(string p, Func includeFileName) + internal sealed class AssemblyLookupLocation(string p, Func includeFileName) { public string Path => p; - public AssemblyPath(string p) : this(p, _ => true) { } + public AssemblyLookupLocation(string p) : this(p, _ => true) { } - public static implicit operator AssemblyPath(string path) => new(path); + public static implicit operator AssemblyLookupLocation(string path) => new(path); /// /// Finds all assemblies nested within the directory `path` @@ -33,19 +33,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } else { - logger.LogInfo($"AssemblyPath: Skipping {dll.FullName}."); + logger.LogInfo($"AssemblyLookupLocation: Skipping {dll.FullName}."); } } } /// - /// Finds all assemblies in `p` that should be indexed and adds them to - /// the list of assembly names to index. + /// Returns a list of paths to all assemblies in `p` that should be indexed. /// - /// List of assembly names to index. /// Logger - public void Process(List dllsToIndex, ILogger logger) + public List GetDlls(ILogger logger) { + var dllsToIndex = new List(); if (File.Exists(p)) { if (includeFileName(System.IO.Path.GetFileName(p))) @@ -54,24 +53,25 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } else { - logger.LogInfo($"AssemblyPath: Skipping {p}."); + logger.LogInfo($"AssemblyLookupLocation: Skipping {p}."); } - return; + return dllsToIndex; } if (Directory.Exists(p)) { - logger.LogInfo($"AssemblyPath: Finding reference DLLs in {p}..."); + logger.LogInfo($"AssemblyLookupLocation: Finding reference DLLs in {p}..."); AddReferenceDirectory(dllsToIndex, logger); } else { - logger.LogInfo("AssemblyCache: Path not found: " + p); + logger.LogInfo("AssemblyLookupLocation: Path not found: " + p); } + return dllsToIndex; } public override bool Equals(object? obj) => - obj is AssemblyPath ap && p.Equals(ap.Path); + obj is AssemblyLookupLocation ap && p.Equals(ap.Path); public override int GetHashCode() => p.GetHashCode(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs index e808235e54d..40f240f3145 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs @@ -12,7 +12,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { public sealed partial class DependencyManager { - private void RestoreNugetPackages(List allNonBinaryFiles, IEnumerable allProjects, IEnumerable allSolutions, HashSet dllPaths) + private void RestoreNugetPackages(List allNonBinaryFiles, IEnumerable allProjects, IEnumerable allSolutions, HashSet dllPaths) { try { @@ -55,7 +55,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } nugetPackageDllPaths.ExceptWith(excludedPaths); - dllPaths.UnionWith(nugetPackageDllPaths.Select(p => new AssemblyPath(p))); + dllPaths.UnionWith(nugetPackageDllPaths.Select(p => new AssemblyLookupLocation(p))); } catch (Exception exc) { @@ -72,7 +72,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching .Paths .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d)) .ToList(); - dllPaths.UnionWith(paths.Select(p => new AssemblyPath(p))); + dllPaths.UnionWith(paths.Select(p => new AssemblyLookupLocation(p))); LogAllUnusedPackages(dependencies); DownloadMissingPackages(allNonBinaryFiles, dllPaths); @@ -148,7 +148,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString())); } - private void DownloadMissingPackages(List allFiles, ISet dllPaths, bool withNugetConfig = true) + private void DownloadMissingPackages(List allFiles, ISet dllPaths, bool withNugetConfig = true) { var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo); var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index 228f26850f8..a59312a1e35 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -91,7 +91,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching this.generatedSources = new(); var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj").ToList(); var allSolutions = allNonBinaryFiles.SelectFileNamesByExtension(".sln").ToList(); - var dllPaths = allFiles.SelectFileNamesByExtension(".dll").Select(x => x).ToHashSet(); + var dllPaths = allFiles.SelectFileNamesByExtension(".dll").Select(x => new AssemblyLookupLocation(x)).ToHashSet(); logger.LogInfo($"Found {allFiles.Count} files, {nonGeneratedSources.Count} source files, {allProjects.Count} project files, {allSolutions.Count} solution files, {dllPaths.Count} DLLs."); @@ -192,7 +192,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching ]); } - private HashSet AddFrameworkDlls(HashSet dllPaths) + private HashSet AddFrameworkDlls(HashSet dllPaths) { var frameworkLocations = new HashSet(); @@ -230,7 +230,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching continue; } - dllPaths.UnionWith(dlls.Select(x => x)); + dllPaths.UnionWith(dlls.Select(x => new AssemblyLookupLocation(x))); frameworkLocations.UnionWith(dlls); } catch (Exception e) @@ -284,7 +284,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet dllPaths, ISet frameworkLocations) + private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet dllPaths, ISet frameworkLocations) { var versionFolders = GetPackageVersionSubDirectories(frameworkPath); if (versionFolders.Length > 1) @@ -313,7 +313,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching .ToArray(); } - private void RemoveFrameworkNugetPackages(ISet dllPaths, int fromIndex = 0) + private void RemoveFrameworkNugetPackages(ISet dllPaths, int fromIndex = 0) { var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks; for (var i = fromIndex; i < packagesInPrioOrder.Length; i++) @@ -322,7 +322,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - private void AddNetFrameworkDlls(ISet dllPaths, ISet frameworkLocations) + private void AddNetFrameworkDlls(ISet dllPaths, ISet frameworkLocations) { // Multiple dotnet framework packages could be present. // The order of the packages is important, we're adding the first one that is present in the nuget cache. @@ -371,7 +371,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (runtimeLocation is null) { runtimeLocation ??= Runtime.ExecutingRuntime; - dllPaths.Add(new AssemblyPath(runtimeLocation, name => !name.StartsWith("Semmle."))); + dllPaths.Add(new AssemblyLookupLocation(runtimeLocation, name => !name.StartsWith("Semmle."))); } else { @@ -382,7 +382,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching frameworkLocations.Add(runtimeLocation); } - private void RemoveNugetPackageReference(string packagePrefix, ISet dllPaths) + private void RemoveNugetPackageReference(string packagePrefix, ISet dllPaths) { var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant(); if (packageFolder == null) @@ -404,7 +404,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return fileContent.IsNewProjectStructureUsed && fileContent.UseAspNetCoreDlls; } - private void AddAspNetCoreFrameworkDlls(ISet dllPaths, ISet frameworkLocations) + private void AddAspNetCoreFrameworkDlls(ISet dllPaths, ISet frameworkLocations) { if (!IsAspNetCoreDetected()) { @@ -426,7 +426,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - private void AddMicrosoftWindowsDesktopDlls(ISet dllPaths, ISet frameworkLocations) + private void AddMicrosoftWindowsDesktopDlls(ISet dllPaths, ISet frameworkLocations) { if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework, packageDirectory) is string windowsDesktopApp) { From 99f0ed26e9791f1bdf2e2e0ea8736e2958f0a73f Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Tue, 9 Apr 2024 13:01:20 +0200 Subject: [PATCH 43/50] C#: Make the assembly lookup case insensitive on the dll file extension and log if no dlls are found in a directory. --- .../AssemblyLookupLocation.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs index 380c60cab56..8711b7f2a6c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using Semmle.Util.Logging; namespace Semmle.Extraction.CSharp.DependencyFetching @@ -25,7 +26,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// The directory to index. private void AddReferenceDirectory(List dllsToIndex, ILogger logger) { - foreach (var dll in new DirectoryInfo(p).EnumerateFiles("*.dll", SearchOption.AllDirectories)) + var dlls = new DirectoryInfo(p).EnumerateFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true, MatchCasing = MatchCasing.CaseInsensitive, AttributesToSkip = FileAttributes.None }); + if (!dlls.Any()) + { + logger.LogWarning($"AssemblyLookupLocation: No DLLs found in the path '{p}'."); + return; + } + foreach (var dll in dlls) { if (includeFileName(dll.Name)) { From a18a4fb62e1a8fc503279a87d9a3f37c03519737 Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Wed, 10 Apr 2024 11:38:27 +0100 Subject: [PATCH 44/50] Avoid magic in `TSynthLocation` definition This improves performance, because in this case magic is not beneficial. --- go/ql/lib/semmle/go/internal/Locations.qll | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/go/ql/lib/semmle/go/internal/Locations.qll b/go/ql/lib/semmle/go/internal/Locations.qll index 498ac9d1170..8b0a206b883 100644 --- a/go/ql/lib/semmle/go/internal/Locations.qll +++ b/go/ql/lib/semmle/go/internal/Locations.qll @@ -11,12 +11,19 @@ newtype TLocation = TSynthLocation(string filepath, int startline, int startcolumn, int endline, int endcolumn) { any(DataFlow::Node n).hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn) and // avoid overlap with existing DB locations - not exists(File f | - locations_default(_, f, startline, startcolumn, endline, endcolumn) and - f.getAbsolutePath() = filepath - ) + not existingDBLocation(filepath, startline, startcolumn, endline, endcolumn) } +pragma[nomagic] +private predicate existingDBLocation( + string filepath, int startline, int startcolumn, int endline, int endcolumn +) { + exists(File f | + locations_default(_, f, startline, startcolumn, endline, endcolumn) and + f.getAbsolutePath() = filepath + ) +} + /** * A location as given by a file, a start line, a start column, * an end line, and an end column. From 3b42dc25a144c8dc64fecbd01075a5b9e399dae0 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Tue, 9 Apr 2024 16:16:21 +0200 Subject: [PATCH 45/50] C#: Also use AssemblyLookupLocation for framework dlls. --- .../AssemblyLookupLocation.cs | 35 +++++++++++-------- .../DependencyManager.cs | 25 ++----------- 2 files changed, 23 insertions(+), 37 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs index 8711b7f2a6c..533b547e45d 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs @@ -10,7 +10,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// Used to represent a path to an assembly or a directory containing assemblies /// and a selector function to determine which files to include, when indexing the assemblies. /// - internal sealed class AssemblyLookupLocation(string p, Func includeFileName) + internal sealed class AssemblyLookupLocation(string p, Func includeFileName, bool indexSubdirectories = true) { public string Path => p; @@ -26,22 +26,29 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// The directory to index. private void AddReferenceDirectory(List dllsToIndex, ILogger logger) { - var dlls = new DirectoryInfo(p).EnumerateFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true, MatchCasing = MatchCasing.CaseInsensitive, AttributesToSkip = FileAttributes.None }); - if (!dlls.Any()) + try { - logger.LogWarning($"AssemblyLookupLocation: No DLLs found in the path '{p}'."); - return; + var dlls = new DirectoryInfo(p).EnumerateFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = indexSubdirectories, MatchCasing = MatchCasing.CaseInsensitive, AttributesToSkip = FileAttributes.None }); + if (!dlls.Any()) + { + logger.LogWarning($"AssemblyLookupLocation: No DLLs found in the path '{p}'."); + return; + } + foreach (var dll in dlls) + { + if (includeFileName(dll.Name)) + { + dllsToIndex.Add(dll.FullName); + } + else + { + logger.LogInfo($"AssemblyLookupLocation: Skipping {dll.FullName}."); + } + } } - foreach (var dll in dlls) + catch (Exception e) { - if (includeFileName(dll.Name)) - { - dllsToIndex.Add(dll.FullName); - } - else - { - logger.LogInfo($"AssemblyLookupLocation: Skipping {dll.FullName}."); - } + logger.LogError($"AssemblyLookupLocation: Error while searching for DLLs in '{p}': {e.Message}"); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index a59312a1e35..8115b0e8ae5 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -214,29 +214,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching continue; } - if (useSubfolders) - { - dllPaths.Add(path); - frameworkLocations.Add(path); - continue; - } - - try - { - var dlls = Directory.GetFiles(path, "*.dll", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive }); - if (dlls.Length == 0) - { - logger.LogError($"No DLLs found in specified framework reference path '{path}'."); - continue; - } - - dllPaths.UnionWith(dlls.Select(x => new AssemblyLookupLocation(x))); - frameworkLocations.UnionWith(dlls); - } - catch (Exception e) - { - logger.LogError($"Error while searching for DLLs in '{path}': {e.Message}"); - } + dllPaths.Add(new AssemblyLookupLocation(path, _ => true, useSubfolders)); + frameworkLocations.Add(path); } return frameworkLocations; From 78ca691912425c515cc6fccce1af49efd8d7b07a Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 10 Apr 2024 13:12:17 +0200 Subject: [PATCH 46/50] Python: remove deprecated points-to test for zope --- .../ql/test/library-tests/web/zope/Test.expected | 3 --- python/ql/test/library-tests/web/zope/Test.ql | 10 ---------- python/ql/test/library-tests/web/zope/options | 1 - python/ql/test/library-tests/web/zope/test.py | 14 -------------- 4 files changed, 28 deletions(-) delete mode 100644 python/ql/test/library-tests/web/zope/Test.expected delete mode 100644 python/ql/test/library-tests/web/zope/Test.ql delete mode 100644 python/ql/test/library-tests/web/zope/options delete mode 100644 python/ql/test/library-tests/web/zope/test.py diff --git a/python/ql/test/library-tests/web/zope/Test.expected b/python/ql/test/library-tests/web/zope/Test.expected deleted file mode 100644 index bb3ca1f441e..00000000000 --- a/python/ql/test/library-tests/web/zope/Test.expected +++ /dev/null @@ -1,3 +0,0 @@ -| 12 | ControlFlowNode for implementer | class implementer | ../../../query-tests/Security/lib/zope/interface/__init__.py:5 | -| 13 | ControlFlowNode for IThing | class IThing | test.py:4 | -| 14 | ControlFlowNode for Thing | class Thing | test.py:9 | diff --git a/python/ql/test/library-tests/web/zope/Test.ql b/python/ql/test/library-tests/web/zope/Test.ql deleted file mode 100644 index ca705ac292e..00000000000 --- a/python/ql/test/library-tests/web/zope/Test.ql +++ /dev/null @@ -1,10 +0,0 @@ -import python -import semmle.python.TestUtils - -from ControlFlowNode f, Value v, ControlFlowNode x -where - exists(ExprStmt s | s.getValue().getAFlowNode() = f) and - f.pointsTo(v, x) and - f.getLocation().getFile().getBaseName() = "test.py" -select f.getLocation().getStartLine(), f.toString(), v.toString(), - remove_library_prefix(x.getLocation()) diff --git a/python/ql/test/library-tests/web/zope/options b/python/ql/test/library-tests/web/zope/options deleted file mode 100644 index 7fb713d5924..00000000000 --- a/python/ql/test/library-tests/web/zope/options +++ /dev/null @@ -1 +0,0 @@ -semmle-extractor-options: --max-import-depth=3 -p ../../../query-tests/Security/lib/ diff --git a/python/ql/test/library-tests/web/zope/test.py b/python/ql/test/library-tests/web/zope/test.py deleted file mode 100644 index 64ac9138d7e..00000000000 --- a/python/ql/test/library-tests/web/zope/test.py +++ /dev/null @@ -1,14 +0,0 @@ - -from zope.interface import Interface, implementer - -class IThing(Interface): - pass - - -@implementer(IThing) -class Thing(object): - pass - -implementer -IThing -Thing From 9615e2ded98443b05a7166687171e6c33fa14f66 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Wed, 10 Apr 2024 13:12:36 +0200 Subject: [PATCH 47/50] Python: Remove deprecated stubs for points-to tests I grep'ed through all our options files, and couldn't find any tests that relies on these anymore :+1: --- .../Security/lib/Crypto/Cipher/AES.py | 3 - .../Security/lib/Crypto/Cipher/ARC4.py | 3 - .../Security/lib/Crypto/Cipher/__init__.py | 1 - .../Security/lib/Crypto/PublicKey/DSA.py | 2 - .../Security/lib/Crypto/PublicKey/RSA.py | 2 - .../Security/lib/Crypto/PublicKey/__init__.py | 0 .../Security/lib/Crypto/__init__.py | 0 .../test/query-tests/Security/lib/airspeed.py | 3 - .../test/query-tests/Security/lib/base64.py | 2 - .../test/query-tests/Security/lib/bottle.py | 69 ------------------- .../Security/lib/cheetah/Template/__init__.py | 2 - .../Security/lib/cherrypy/__init__.py | 15 ---- .../Security/lib/cherrypy/_helper.py | 31 --------- .../test/query-tests/Security/lib/chevron.py | 6 -- .../Security/lib/cryptography/__init__.py | 0 .../lib/cryptography/hazmat/__init__.py | 0 .../hazmat/primitives/__init__.py | 0 .../hazmat/primitives/asymmetric/__init__.py | 0 .../hazmat/primitives/asymmetric/dsa.py | 4 -- .../hazmat/primitives/asymmetric/ec.py | 22 ------ .../hazmat/primitives/asymmetric/rsa.py | 3 - .../hazmat/primitives/ciphers/__init__.py | 3 - .../hazmat/primitives/ciphers/algorithms.py | 3 - .../ql/test/query-tests/Security/lib/dill.py | 2 - .../Security/lib/django/__init__.py | 0 .../Security/lib/django/conf/__init__.py | 0 .../Security/lib/django/conf/urls.py | 7 -- .../Security/lib/django/db/__init__.py | 1 - .../Security/lib/django/db/models/__init__.py | 2 - .../lib/django/db/models/expressions.py | 2 - .../Security/lib/django/http/__init__.py | 2 - .../Security/lib/django/http/request.py | 2 - .../Security/lib/django/http/response.py | 46 ------------- .../Security/lib/django/shortcuts.py | 8 --- .../query-tests/Security/lib/django/urls.py | 7 -- .../Security/lib/django/views/__init__.py | 3 - .../Security/lib/django/views/generic.py | 3 - .../Security/lib/fabric/__init__.py | 3 - .../query-tests/Security/lib/fabric/api.py | 29 -------- .../Security/lib/fabric/connection.py | 15 ---- .../query-tests/Security/lib/fabric/group.py | 11 --- .../query-tests/Security/lib/fabric/tasks.py | 2 - .../Security/lib/falcon/__init__.py | 4 -- .../query-tests/Security/lib/falcon/api.py | 14 ---- .../Security/lib/falcon/request.py | 3 - .../Security/lib/falcon/response.py | 4 -- .../Security/lib/flask/__init__.py | 33 --------- .../query-tests/Security/lib/flask/globals.py | 6 -- .../query-tests/Security/lib/flask/urls.py | 0 .../query-tests/Security/lib/flask/views.py | 6 -- .../Security/lib/invoke/__init__.py | 8 --- .../Security/lib/invoke/context.py | 3 - .../query-tests/Security/lib/invoke/tasks.py | 2 - .../test/query-tests/Security/lib/jinja2.py | 23 ------- .../Security/lib/libxml2/__init__.py | 10 --- .../query-tests/Security/lib/lxml/__init__.py | 0 .../query-tests/Security/lib/lxml/etree.py | 37 ---------- .../test/query-tests/Security/lib/marshall.py | 2 - .../query-tests/Security/lib/os/__init__.py | 17 ----- .../test/query-tests/Security/lib/os/path.py | 5 -- .../Security/lib/paramiko/__init__.py | 0 .../Security/lib/paramiko/client.py | 15 ---- .../test/query-tests/Security/lib/pickle.py | 2 - .../query-tests/Security/lib/pyOpenSSL/SSL.py | 10 --- .../Security/lib/pyOpenSSL/__init__.py | 0 .../Security/lib/pyramid/__init__.py | 0 .../Security/lib/pyramid/response.py | 2 - .../query-tests/Security/lib/pyramid/view.py | 7 -- .../test/query-tests/Security/lib/requests.py | 21 ------ .../query-tests/Security/lib/subprocess.py | 2 - python/ql/test/query-tests/Security/lib/tg.py | 30 -------- .../Security/lib/tornado/__init__.py | 0 .../query-tests/Security/lib/tornado/web.py | 2 - .../query-tests/Security/lib/traceback.py | 2 - .../Security/lib/twisted/__init__.py | 0 .../Security/lib/twisted/web/__init__.py | 0 .../Security/lib/twisted/web/http.py | 2 - .../Security/lib/twisted/web/resource.py | 2 - .../ql/test/query-tests/Security/lib/yaml.py | 2 - .../query-tests/Security/lib/zope/__init__.py | 0 .../Security/lib/zope/interface/__init__.py | 9 --- 81 files changed, 604 deletions(-) delete mode 100644 python/ql/test/query-tests/Security/lib/Crypto/Cipher/AES.py delete mode 100644 python/ql/test/query-tests/Security/lib/Crypto/Cipher/ARC4.py delete mode 100644 python/ql/test/query-tests/Security/lib/Crypto/Cipher/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/Crypto/PublicKey/DSA.py delete mode 100644 python/ql/test/query-tests/Security/lib/Crypto/PublicKey/RSA.py delete mode 100644 python/ql/test/query-tests/Security/lib/Crypto/PublicKey/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/Crypto/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/airspeed.py delete mode 100644 python/ql/test/query-tests/Security/lib/base64.py delete mode 100644 python/ql/test/query-tests/Security/lib/bottle.py delete mode 100644 python/ql/test/query-tests/Security/lib/cheetah/Template/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/cherrypy/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/cherrypy/_helper.py delete mode 100644 python/ql/test/query-tests/Security/lib/chevron.py delete mode 100644 python/ql/test/query-tests/Security/lib/cryptography/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/cryptography/hazmat/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/dsa.py delete mode 100644 python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/ec.py delete mode 100644 python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/rsa.py delete mode 100644 python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/ciphers/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/ciphers/algorithms.py delete mode 100644 python/ql/test/query-tests/Security/lib/dill.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/conf/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/conf/urls.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/db/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/db/models/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/db/models/expressions.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/http/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/http/request.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/http/response.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/shortcuts.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/urls.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/views/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/django/views/generic.py delete mode 100644 python/ql/test/query-tests/Security/lib/fabric/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/fabric/api.py delete mode 100644 python/ql/test/query-tests/Security/lib/fabric/connection.py delete mode 100644 python/ql/test/query-tests/Security/lib/fabric/group.py delete mode 100644 python/ql/test/query-tests/Security/lib/fabric/tasks.py delete mode 100644 python/ql/test/query-tests/Security/lib/falcon/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/falcon/api.py delete mode 100644 python/ql/test/query-tests/Security/lib/falcon/request.py delete mode 100644 python/ql/test/query-tests/Security/lib/falcon/response.py delete mode 100644 python/ql/test/query-tests/Security/lib/flask/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/flask/globals.py delete mode 100644 python/ql/test/query-tests/Security/lib/flask/urls.py delete mode 100644 python/ql/test/query-tests/Security/lib/flask/views.py delete mode 100644 python/ql/test/query-tests/Security/lib/invoke/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/invoke/context.py delete mode 100644 python/ql/test/query-tests/Security/lib/invoke/tasks.py delete mode 100644 python/ql/test/query-tests/Security/lib/jinja2.py delete mode 100644 python/ql/test/query-tests/Security/lib/libxml2/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/lxml/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/lxml/etree.py delete mode 100644 python/ql/test/query-tests/Security/lib/marshall.py delete mode 100644 python/ql/test/query-tests/Security/lib/os/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/os/path.py delete mode 100644 python/ql/test/query-tests/Security/lib/paramiko/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/paramiko/client.py delete mode 100644 python/ql/test/query-tests/Security/lib/pickle.py delete mode 100644 python/ql/test/query-tests/Security/lib/pyOpenSSL/SSL.py delete mode 100644 python/ql/test/query-tests/Security/lib/pyOpenSSL/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/pyramid/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/pyramid/response.py delete mode 100644 python/ql/test/query-tests/Security/lib/pyramid/view.py delete mode 100644 python/ql/test/query-tests/Security/lib/requests.py delete mode 100644 python/ql/test/query-tests/Security/lib/subprocess.py delete mode 100644 python/ql/test/query-tests/Security/lib/tg.py delete mode 100644 python/ql/test/query-tests/Security/lib/tornado/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/tornado/web.py delete mode 100644 python/ql/test/query-tests/Security/lib/traceback.py delete mode 100644 python/ql/test/query-tests/Security/lib/twisted/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/twisted/web/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/twisted/web/http.py delete mode 100644 python/ql/test/query-tests/Security/lib/twisted/web/resource.py delete mode 100644 python/ql/test/query-tests/Security/lib/yaml.py delete mode 100644 python/ql/test/query-tests/Security/lib/zope/__init__.py delete mode 100644 python/ql/test/query-tests/Security/lib/zope/interface/__init__.py diff --git a/python/ql/test/query-tests/Security/lib/Crypto/Cipher/AES.py b/python/ql/test/query-tests/Security/lib/Crypto/Cipher/AES.py deleted file mode 100644 index bcab58405ea..00000000000 --- a/python/ql/test/query-tests/Security/lib/Crypto/Cipher/AES.py +++ /dev/null @@ -1,3 +0,0 @@ - -def new(*args): - pass diff --git a/python/ql/test/query-tests/Security/lib/Crypto/Cipher/ARC4.py b/python/ql/test/query-tests/Security/lib/Crypto/Cipher/ARC4.py deleted file mode 100644 index bcab58405ea..00000000000 --- a/python/ql/test/query-tests/Security/lib/Crypto/Cipher/ARC4.py +++ /dev/null @@ -1,3 +0,0 @@ - -def new(*args): - pass diff --git a/python/ql/test/query-tests/Security/lib/Crypto/Cipher/__init__.py b/python/ql/test/query-tests/Security/lib/Crypto/Cipher/__init__.py deleted file mode 100644 index ab22f65be6e..00000000000 --- a/python/ql/test/query-tests/Security/lib/Crypto/Cipher/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__ = ['AES', 'ARC4'] diff --git a/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/DSA.py b/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/DSA.py deleted file mode 100644 index e002a77697e..00000000000 --- a/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/DSA.py +++ /dev/null @@ -1,2 +0,0 @@ -def generate(bits): - pass diff --git a/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/RSA.py b/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/RSA.py deleted file mode 100644 index e002a77697e..00000000000 --- a/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/RSA.py +++ /dev/null @@ -1,2 +0,0 @@ -def generate(bits): - pass diff --git a/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/__init__.py b/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/Crypto/__init__.py b/python/ql/test/query-tests/Security/lib/Crypto/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/airspeed.py b/python/ql/test/query-tests/Security/lib/airspeed.py deleted file mode 100644 index aea81c72b71..00000000000 --- a/python/ql/test/query-tests/Security/lib/airspeed.py +++ /dev/null @@ -1,3 +0,0 @@ -class Template: - def __init__(self, content, filename=""): - pass diff --git a/python/ql/test/query-tests/Security/lib/base64.py b/python/ql/test/query-tests/Security/lib/base64.py deleted file mode 100644 index a2b97ca63ac..00000000000 --- a/python/ql/test/query-tests/Security/lib/base64.py +++ /dev/null @@ -1,2 +0,0 @@ -def decodestring(s): - return None diff --git a/python/ql/test/query-tests/Security/lib/bottle.py b/python/ql/test/query-tests/Security/lib/bottle.py deleted file mode 100644 index fe42507852c..00000000000 --- a/python/ql/test/query-tests/Security/lib/bottle.py +++ /dev/null @@ -1,69 +0,0 @@ - -class Bottle(object): - - def route(self, path=None, method='GET', **options): - pass - - def get(self, path=None, method='GET', **options): - """ Equals :meth:`route`. """ - return self.route(path, method, **options) - - def post(self, path=None, method='POST', **options): - """ Equals :meth:`route` with a ``POST`` method parameter. """ - return self.route(path, method, **options) - - def put(self, path=None, method='PUT', **options): - """ Equals :meth:`route` with a ``PUT`` method parameter. """ - return self.route(path, method, **options) - - def delete(self, path=None, method='DELETE', **options): - """ Equals :meth:`route` with a ``DELETE`` method parameter. """ - return self.route(path, method, **options) - - def error(self, code=500): - """ Decorator: Register an output handler for a HTTP error code""" - def wrapper(handler): - self.error_handler[int(code)] = handler - return handler - return wrapper - -#Use same wrapper logic as the original `bottle` code. - -def make_default_app_wrapper(name): - """ Return a callable that relays calls to the current default app. """ - - @functools.wraps(getattr(Bottle, name)) - def wrapper(*a, **ka): - return getattr(app(), name)(*a, **ka) - - return wrapper - -route = make_default_app_wrapper('route') -get = make_default_app_wrapper('get') -post = make_default_app_wrapper('post') -put = make_default_app_wrapper('put') -delete = make_default_app_wrapper('delete') -patch = make_default_app_wrapper('patch') -error = make_default_app_wrapper('error') -mount = make_default_app_wrapper('mount') -hook = make_default_app_wrapper('hook') -install = make_default_app_wrapper('install') -uninstall = make_default_app_wrapper('uninstall') -url = make_default_app_wrapper('get_url') - -class LocalProxy(object): - pass - -class LocalRequest(LocalProxy): - pass - -class LocalResponse(LocalProxy): - pass - - -request = LocalRequest() -response = LocalResponse() - - -def redirect(url, code=None): - pass diff --git a/python/ql/test/query-tests/Security/lib/cheetah/Template/__init__.py b/python/ql/test/query-tests/Security/lib/cheetah/Template/__init__.py deleted file mode 100644 index 04b862eafea..00000000000 --- a/python/ql/test/query-tests/Security/lib/cheetah/Template/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -class Template(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/cherrypy/__init__.py b/python/ql/test/query-tests/Security/lib/cherrypy/__init__.py deleted file mode 100644 index a1ae780e3b5..00000000000 --- a/python/ql/test/query-tests/Security/lib/cherrypy/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ - - -from ._helper import expose, popargs, url - -class _ThreadLocalProxy(object): - def __getattr__(self, name): - pass - - -request = _ThreadLocalProxy('request') -response = _ThreadLocalProxy('response') - -def quickstart(root=None, script_name='', config=None): - """Mount the given root, start the builtin server (and engine), then block.""" - pass diff --git a/python/ql/test/query-tests/Security/lib/cherrypy/_helper.py b/python/ql/test/query-tests/Security/lib/cherrypy/_helper.py deleted file mode 100644 index fc3f3604a57..00000000000 --- a/python/ql/test/query-tests/Security/lib/cherrypy/_helper.py +++ /dev/null @@ -1,31 +0,0 @@ -def expose(func=None, alias=None): - """Expose the function or class. - Optionally provide an alias or set of aliases. - """ - def expose_(func): - func.exposed = True - return func - - return expose_ - - -def popargs(*args, **kwargs): - """Decorate _cp_dispatch.""" - - def decorated(cls_or_self=None, vpath=None): - if inspect.isclass(cls_or_self): - # cherrypy.popargs is a class decorator - return cls - - # We're in the actual function - self = cls_or_self - if vpath: - return getattr(self, vpath.pop(0), None) - else: - return self - - return decorated - -def url(path='', qs='', script_name=None, base=None, relative=None): - #Do some opaque stuff here... - return new_url diff --git a/python/ql/test/query-tests/Security/lib/chevron.py b/python/ql/test/query-tests/Security/lib/chevron.py deleted file mode 100644 index 012ba58fdb8..00000000000 --- a/python/ql/test/query-tests/Security/lib/chevron.py +++ /dev/null @@ -1,6 +0,0 @@ - - -def render(template='', data={}, partials_path='.', partials_ext='mustache', - partials_dict={}, padding='', def_ldel='{{', def_rdel='}}', - scopes=None): - pass diff --git a/python/ql/test/query-tests/Security/lib/cryptography/__init__.py b/python/ql/test/query-tests/Security/lib/cryptography/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/__init__.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/__init__.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/__init__.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/dsa.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/dsa.py deleted file mode 100644 index b4c37f6b540..00000000000 --- a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/dsa.py +++ /dev/null @@ -1,4 +0,0 @@ - -def generate_private_key(key_size, backend): - pass - diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/ec.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/ec.py deleted file mode 100644 index d2eb4b0f1af..00000000000 --- a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/ec.py +++ /dev/null @@ -1,22 +0,0 @@ - -def generate_private_key(curve, backend): - pass - -class SECT571R1(object): - name = "sect571r1" - key_size = 570 - - -class SECP384R1(object): - name = "secp384r1" - key_size = 384 - - -class SECP224R1(object): - name = "secp224r1" - key_size = 224 - - -class SECT163K1(object): - name = "sect163k1" - key_size = 163 \ No newline at end of file diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/rsa.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/rsa.py deleted file mode 100644 index 3a5c91734c2..00000000000 --- a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/rsa.py +++ /dev/null @@ -1,3 +0,0 @@ - -def generate_private_key(public_exponent, key_size, backend): - pass diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/ciphers/__init__.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/ciphers/__init__.py deleted file mode 100644 index f37f14e88f8..00000000000 --- a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/ciphers/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ - -class Cipher(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/ciphers/algorithms.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/ciphers/algorithms.py deleted file mode 100644 index e423804e4f5..00000000000 --- a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/ciphers/algorithms.py +++ /dev/null @@ -1,3 +0,0 @@ - -class ARC4(object): - name = "RC4" diff --git a/python/ql/test/query-tests/Security/lib/dill.py b/python/ql/test/query-tests/Security/lib/dill.py deleted file mode 100644 index 410fa21087e..00000000000 --- a/python/ql/test/query-tests/Security/lib/dill.py +++ /dev/null @@ -1,2 +0,0 @@ -def loads(*args, **kwargs): - return None diff --git a/python/ql/test/query-tests/Security/lib/django/__init__.py b/python/ql/test/query-tests/Security/lib/django/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/django/conf/__init__.py b/python/ql/test/query-tests/Security/lib/django/conf/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/django/conf/urls.py b/python/ql/test/query-tests/Security/lib/django/conf/urls.py deleted file mode 100644 index 49a248e16aa..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/conf/urls.py +++ /dev/null @@ -1,7 +0,0 @@ -# https://docs.djangoproject.com/en/1.11/_modules/django/conf/urls/#url -def url(regex, view, kwargs=None, name=None): - pass - - -def patterns(*urls): - pass diff --git a/python/ql/test/query-tests/Security/lib/django/db/__init__.py b/python/ql/test/query-tests/Security/lib/django/db/__init__.py deleted file mode 100644 index 3e291c5731d..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/db/__init__.py +++ /dev/null @@ -1 +0,0 @@ -connection = object() diff --git a/python/ql/test/query-tests/Security/lib/django/db/models/__init__.py b/python/ql/test/query-tests/Security/lib/django/db/models/__init__.py deleted file mode 100644 index eb9c72adc45..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/db/models/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -class Model: - pass diff --git a/python/ql/test/query-tests/Security/lib/django/db/models/expressions.py b/python/ql/test/query-tests/Security/lib/django/db/models/expressions.py deleted file mode 100644 index d7e0d1c27b6..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/db/models/expressions.py +++ /dev/null @@ -1,2 +0,0 @@ -class RawSQL: - pass diff --git a/python/ql/test/query-tests/Security/lib/django/http/__init__.py b/python/ql/test/query-tests/Security/lib/django/http/__init__.py deleted file mode 100644 index f2ac6d2c55b..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/http/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .response import * -from .request import HttpRequest diff --git a/python/ql/test/query-tests/Security/lib/django/http/request.py b/python/ql/test/query-tests/Security/lib/django/http/request.py deleted file mode 100644 index 93afdaf29b7..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/http/request.py +++ /dev/null @@ -1,2 +0,0 @@ -class HttpRequest(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/django/http/response.py b/python/ql/test/query-tests/Security/lib/django/http/response.py deleted file mode 100644 index 1110a3cde19..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/http/response.py +++ /dev/null @@ -1,46 +0,0 @@ -class HttpResponseBase(object): - status_code = 200 - - def __init__(self, content_type=None, status=None, reason=None, charset=None): - pass - - -class HttpResponse(HttpResponseBase): - def __init__(self, content=b"", *args, **kwargs): - super().__init__(*args, **kwargs) - # Content is a bytestring. See the `content` property methods. - self.content = content - - -class HttpResponseRedirectBase(HttpResponse): - def __init__(self, redirect_to, *args, **kwargs): - super().__init__(*args, **kwargs) - ... - - -class HttpResponsePermanentRedirect(HttpResponseRedirectBase): - status_code = 301 - - -class HttpResponseRedirect(HttpResponseRedirectBase): - status_code = 302 - - -class HttpResponseNotFound(HttpResponse): - status_code = 404 - - -class JsonResponse(HttpResponse): - - def __init__( - self, - data, - encoder=..., - safe=True, - json_dumps_params=None, - **kwargs - ): - # fake code to represent what is going on :) - kwargs.setdefault("content_type", "application/json") - data = str(data) - super().__init__(content=data, **kwargs) diff --git a/python/ql/test/query-tests/Security/lib/django/shortcuts.py b/python/ql/test/query-tests/Security/lib/django/shortcuts.py deleted file mode 100644 index 4783729178a..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/shortcuts.py +++ /dev/null @@ -1,8 +0,0 @@ -# see https://docs.djangoproject.com/en/1.11/_modules/django/shortcuts/#redirect -# https://github.com/django/django/blob/86908785076b2bbc31b908781da6b6ad1779b18b/django/shortcuts.py - -def render(request, template_name, context=None, content_type=None, status=None, using=None): - pass - -def redirect(to, *args, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/django/urls.py b/python/ql/test/query-tests/Security/lib/django/urls.py deleted file mode 100644 index dc28eece874..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/urls.py +++ /dev/null @@ -1,7 +0,0 @@ -from functools import partial - -def _path(route, view, kwargs=None, name=None, Pattern=None): - pass - -path = partial(_path, Pattern='RoutePattern (but this is a mock)') -re_path = partial(_path, Pattern='RegexPattern (but this is a mock)') diff --git a/python/ql/test/query-tests/Security/lib/django/views/__init__.py b/python/ql/test/query-tests/Security/lib/django/views/__init__.py deleted file mode 100644 index 47de98f1351..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/views/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# For django 2.x and 3.x -class View: - pass diff --git a/python/ql/test/query-tests/Security/lib/django/views/generic.py b/python/ql/test/query-tests/Security/lib/django/views/generic.py deleted file mode 100644 index edb71bed762..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/views/generic.py +++ /dev/null @@ -1,3 +0,0 @@ -# For django 1.x -class View: - pass diff --git a/python/ql/test/query-tests/Security/lib/fabric/__init__.py b/python/ql/test/query-tests/Security/lib/fabric/__init__.py deleted file mode 100644 index fcdb406611e..00000000000 --- a/python/ql/test/query-tests/Security/lib/fabric/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .connection import Connection -from .group import Group, SerialGroup, ThreadingGroup -from .tasks import task diff --git a/python/ql/test/query-tests/Security/lib/fabric/api.py b/python/ql/test/query-tests/Security/lib/fabric/api.py deleted file mode 100644 index b26d5a04b6d..00000000000 --- a/python/ql/test/query-tests/Security/lib/fabric/api.py +++ /dev/null @@ -1,29 +0,0 @@ -# For the 1.x version - -def needs_host(func): - @wraps(func) - def inner(*args, **kwargs): - return func(*args, **kwargs) - return inner - - -def local(command, capture=False, shell=None): - pass - - -@needs_host -def run(command, shell=True, pty=True, combine_stderr=None, quiet=False, - warn_only=False, stdout=None, stderr=None, timeout=None, shell_escape=None, - capture_buffer_size=None): - pass - - -@needs_host -def sudo(command, shell=True, pty=True, combine_stderr=None, user=None, - quiet=False, warn_only=False, stdout=None, stderr=None, group=None, - timeout=None, shell_escape=None, capture_buffer_size=None): - pass - -# https://github.com/fabric/fabric/blob/1.14/fabric/tasks.py#L281 -def execute(task, *args, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/fabric/connection.py b/python/ql/test/query-tests/Security/lib/fabric/connection.py deleted file mode 100644 index e046aa4e744..00000000000 --- a/python/ql/test/query-tests/Security/lib/fabric/connection.py +++ /dev/null @@ -1,15 +0,0 @@ -from invoke import Context - -@decorator -def opens(method, self, *args, **kwargs): - self.open() - return method(self, *args, **kwargs) - -class Connection(Context): - - def open(self): - pass - - @opens - def run(self, command, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/fabric/group.py b/python/ql/test/query-tests/Security/lib/fabric/group.py deleted file mode 100644 index 131231d2268..00000000000 --- a/python/ql/test/query-tests/Security/lib/fabric/group.py +++ /dev/null @@ -1,11 +0,0 @@ -class Group(list): - def run(self, *args, **kwargs): - raise NotImplementedError - -class SerialGroup(Group): - def run(self, *args, **kwargs): - pass - -class ThreadingGroup(Group): - def run(self, *args, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/fabric/tasks.py b/python/ql/test/query-tests/Security/lib/fabric/tasks.py deleted file mode 100644 index 08968e68ab5..00000000000 --- a/python/ql/test/query-tests/Security/lib/fabric/tasks.py +++ /dev/null @@ -1,2 +0,0 @@ -def task(*args, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/falcon/__init__.py b/python/ql/test/query-tests/Security/lib/falcon/__init__.py deleted file mode 100644 index 5983e6a41a5..00000000000 --- a/python/ql/test/query-tests/Security/lib/falcon/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ - -from falcon.api import API -from falcon.request import Request -from falcon.response import Response diff --git a/python/ql/test/query-tests/Security/lib/falcon/api.py b/python/ql/test/query-tests/Security/lib/falcon/api.py deleted file mode 100644 index 2763d8ca3e1..00000000000 --- a/python/ql/test/query-tests/Security/lib/falcon/api.py +++ /dev/null @@ -1,14 +0,0 @@ - -"""Falcon API class.""" - -class API(object): - - def add_route(self, uri_template, resource, **kwargs): - pass - - def add_sink(self, sink, prefix=r'/'): - pass - - def add_error_handler(self, exception, handler=None): - pass - diff --git a/python/ql/test/query-tests/Security/lib/falcon/request.py b/python/ql/test/query-tests/Security/lib/falcon/request.py deleted file mode 100644 index 4e39d52d841..00000000000 --- a/python/ql/test/query-tests/Security/lib/falcon/request.py +++ /dev/null @@ -1,3 +0,0 @@ - -class Request(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/falcon/response.py b/python/ql/test/query-tests/Security/lib/falcon/response.py deleted file mode 100644 index d03bbee54ac..00000000000 --- a/python/ql/test/query-tests/Security/lib/falcon/response.py +++ /dev/null @@ -1,4 +0,0 @@ - - -class Response(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/flask/__init__.py b/python/ql/test/query-tests/Security/lib/flask/__init__.py deleted file mode 100644 index 73ce61e7ccf..00000000000 --- a/python/ql/test/query-tests/Security/lib/flask/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -from .globals import request -from .globals import current_app - -class Flask(object): - # Only some methods mocked, signature copied from - # https://flask.palletsprojects.com/en/1.1.x/api/#flask.Flask - def run(host=None, port=None, debug=None, load_dotenv=True, **options): - pass - - def make_response(rv): - pass - - def add_url_rule(rule, endpoint=None, view_func=None, provide_automatic_options=None, **options): - pass - -class Response(object): - pass - -def redirect(location, code=302, Response=None): - pass - -def make_response(rv): - if not args: - return current_app.response_class() - if len(args) == 1: - args = args[0] - return current_app.make_response(args) - -def escape(txt): - return Markup.escape(txt) - -def render_template_string(source, **context): - pass \ No newline at end of file diff --git a/python/ql/test/query-tests/Security/lib/flask/globals.py b/python/ql/test/query-tests/Security/lib/flask/globals.py deleted file mode 100644 index 36320cb4886..00000000000 --- a/python/ql/test/query-tests/Security/lib/flask/globals.py +++ /dev/null @@ -1,6 +0,0 @@ - -class LocalProxy(object): - pass - -request = LocalProxy() -current_app = LocalProxy() diff --git a/python/ql/test/query-tests/Security/lib/flask/urls.py b/python/ql/test/query-tests/Security/lib/flask/urls.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/flask/views.py b/python/ql/test/query-tests/Security/lib/flask/views.py deleted file mode 100644 index 1b7007f20da..00000000000 --- a/python/ql/test/query-tests/Security/lib/flask/views.py +++ /dev/null @@ -1,6 +0,0 @@ -class View(object): - pass - - -class MethodView(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/invoke/__init__.py b/python/ql/test/query-tests/Security/lib/invoke/__init__.py deleted file mode 100644 index 1cb865d16ff..00000000000 --- a/python/ql/test/query-tests/Security/lib/invoke/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from .context import Context -from .tasks import task - -def run(command, **kwargs): - pass - -def sudo(command, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/invoke/context.py b/python/ql/test/query-tests/Security/lib/invoke/context.py deleted file mode 100644 index dff247b23f1..00000000000 --- a/python/ql/test/query-tests/Security/lib/invoke/context.py +++ /dev/null @@ -1,3 +0,0 @@ -class Context(object): - def run(self, command, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/invoke/tasks.py b/python/ql/test/query-tests/Security/lib/invoke/tasks.py deleted file mode 100644 index 08968e68ab5..00000000000 --- a/python/ql/test/query-tests/Security/lib/invoke/tasks.py +++ /dev/null @@ -1,2 +0,0 @@ -def task(*args, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/jinja2.py b/python/ql/test/query-tests/Security/lib/jinja2.py deleted file mode 100644 index 7c95c1417ab..00000000000 --- a/python/ql/test/query-tests/Security/lib/jinja2.py +++ /dev/null @@ -1,23 +0,0 @@ - -class Environment(object): - - def __init__(self, loader, autoescape): - pass - -class Template(object): - - def __init__(self, source, autoescape): - pass - -def select_autoescape(files=[]): - def autoescape(template_name): - pass - return autoescape - -class FileSystemLoader(object): - - def __init__(self, searchpath): - pass - -def from_string(source, globals=None, template_class=None): - pass \ No newline at end of file diff --git a/python/ql/test/query-tests/Security/lib/libxml2/__init__.py b/python/ql/test/query-tests/Security/lib/libxml2/__init__.py deleted file mode 100644 index 057488829f4..00000000000 --- a/python/ql/test/query-tests/Security/lib/libxml2/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -def parseFile(filename): - return xmlDoc(_obj=None) - - -class xmlDoc(Object): - def __init__(self, _obj=None): - pass - - def xpathEval(self, expr): - pass diff --git a/python/ql/test/query-tests/Security/lib/lxml/__init__.py b/python/ql/test/query-tests/Security/lib/lxml/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/lxml/etree.py b/python/ql/test/query-tests/Security/lib/lxml/etree.py deleted file mode 100644 index 139553b0d6c..00000000000 --- a/python/ql/test/query-tests/Security/lib/lxml/etree.py +++ /dev/null @@ -1,37 +0,0 @@ -class _ElementTree(object): - def xpath(self, _path, namespaces=None, extensions=None, smart_strings=True, **_variables): - pass - - def xslt(self, _xslt, extensions=None, access_control=None, **_kw): - pass - - -class ETXPath(object): - def __init__(self, path, extensions=None, regexp=True, smart_strings=True): - pass - - -class XPath(object): - def __init__(self, path, namespaces=None, extensions=None, regexp=True, smart_strings=True): - pass - - -class XSLT(object): - def __init__(self, xslt_input, extensions=None, regexp=True, access_control=None): - pass - - -def parse(self, parser=None, base_url=None): - return _ElementTree() - - -def fromstring(self, text, parser=None, base_url=None): - pass - - -def fromstringlist(self, strings, parser=None): - pass - - -def XML(self, text, parser=None, base_url=None): - pass diff --git a/python/ql/test/query-tests/Security/lib/marshall.py b/python/ql/test/query-tests/Security/lib/marshall.py deleted file mode 100644 index 410fa21087e..00000000000 --- a/python/ql/test/query-tests/Security/lib/marshall.py +++ /dev/null @@ -1,2 +0,0 @@ -def loads(*args, **kwargs): - return None diff --git a/python/ql/test/query-tests/Security/lib/os/__init__.py b/python/ql/test/query-tests/Security/lib/os/__init__.py deleted file mode 100644 index 397e2eb12da..00000000000 --- a/python/ql/test/query-tests/Security/lib/os/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -def system(cmd, *args, **kwargs): - return None - -def popen(cmd, *args, **kwargs): - return None - -def chmod(path, mode): - pass - -def open(path, flags, mode): - pass - -def tempnam(*args): - pass - -def tmpnam(*args): - pass diff --git a/python/ql/test/query-tests/Security/lib/os/path.py b/python/ql/test/query-tests/Security/lib/os/path.py deleted file mode 100644 index d54092e80b0..00000000000 --- a/python/ql/test/query-tests/Security/lib/os/path.py +++ /dev/null @@ -1,5 +0,0 @@ -def join(a, *b): - return a + "/" + "/".join(b) - -def normpath(x): - return x diff --git a/python/ql/test/query-tests/Security/lib/paramiko/__init__.py b/python/ql/test/query-tests/Security/lib/paramiko/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/paramiko/client.py b/python/ql/test/query-tests/Security/lib/paramiko/client.py deleted file mode 100644 index 6343fbe78dd..00000000000 --- a/python/ql/test/query-tests/Security/lib/paramiko/client.py +++ /dev/null @@ -1,15 +0,0 @@ -class SSHClient(object): - def __init__(self, *args, **kwargs): - pass - - def set_missing_host_key_policy(self, *args, **kwargs): - pass - -class AutoAddPolicy(object): - pass - -class WarningPolicy(object): - pass - -class RejectPolicy(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/pickle.py b/python/ql/test/query-tests/Security/lib/pickle.py deleted file mode 100644 index 410fa21087e..00000000000 --- a/python/ql/test/query-tests/Security/lib/pickle.py +++ /dev/null @@ -1,2 +0,0 @@ -def loads(*args, **kwargs): - return None diff --git a/python/ql/test/query-tests/Security/lib/pyOpenSSL/SSL.py b/python/ql/test/query-tests/Security/lib/pyOpenSSL/SSL.py deleted file mode 100644 index eb135d2d5e0..00000000000 --- a/python/ql/test/query-tests/Security/lib/pyOpenSSL/SSL.py +++ /dev/null @@ -1,10 +0,0 @@ -SSLv2_METHOD = 1 -SSLv3_METHOD = 2 -SSLv23_METHOD = 3 -TLSv1_METHOD = 4 -TLSv1_1_METHOD = 5 -TLSv1_2_METHOD = 6 - -class Context(object): - def __init__(self, *args, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/pyOpenSSL/__init__.py b/python/ql/test/query-tests/Security/lib/pyOpenSSL/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/pyramid/__init__.py b/python/ql/test/query-tests/Security/lib/pyramid/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/pyramid/response.py b/python/ql/test/query-tests/Security/lib/pyramid/response.py deleted file mode 100644 index a482ac919ff..00000000000 --- a/python/ql/test/query-tests/Security/lib/pyramid/response.py +++ /dev/null @@ -1,2 +0,0 @@ -class Response(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/pyramid/view.py b/python/ql/test/query-tests/Security/lib/pyramid/view.py deleted file mode 100644 index c7487bf827b..00000000000 --- a/python/ql/test/query-tests/Security/lib/pyramid/view.py +++ /dev/null @@ -1,7 +0,0 @@ -# https://docs.pylonsproject.org/projects/pyramid/en/1.10-branch/_modules/pyramid/view.html#view_config -class view_config(object): - def __init__(self, **settings): - pass - - def __call__(self, wrapped): - pass diff --git a/python/ql/test/query-tests/Security/lib/requests.py b/python/ql/test/query-tests/Security/lib/requests.py deleted file mode 100644 index 01afdbb1208..00000000000 --- a/python/ql/test/query-tests/Security/lib/requests.py +++ /dev/null @@ -1,21 +0,0 @@ - -def get(url, params=None, **kwargs): - pass - -def options(url, **kwargs): - pass - -def head(url, **kwargs): - pass - -def post(url, data=None, json=None, **kwargs): - pass - -def put(url, data=None, **kwargs): - pass - -def patch(url, data=None, **kwargs): - pass - -def delete(url, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/subprocess.py b/python/ql/test/query-tests/Security/lib/subprocess.py deleted file mode 100644 index efb2ba183f0..00000000000 --- a/python/ql/test/query-tests/Security/lib/subprocess.py +++ /dev/null @@ -1,2 +0,0 @@ -def Popen(*args, **kwargs): - return None \ No newline at end of file diff --git a/python/ql/test/query-tests/Security/lib/tg.py b/python/ql/test/query-tests/Security/lib/tg.py deleted file mode 100644 index 976b7427b33..00000000000 --- a/python/ql/test/query-tests/Security/lib/tg.py +++ /dev/null @@ -1,30 +0,0 @@ - -def validate(validators): - def validator(func): - return func - -def expose(*args): - if cond: - return args[0] - def with_template(func): - func - return with_template - -class TGController(object): - pass - -class TurboGearsContextMember(object): - """Member of the TurboGears request context. - Provides access to turbogears context members - like request, response, template context and so on - """ - - def __init__(self, name): - self.__dict__['name'] = name - - def _current_obj(self): - return getattr(context, self.name) - - -request = TurboGearsContextMember(name="request") -response = TurboGearsContextMember(name="response") diff --git a/python/ql/test/query-tests/Security/lib/tornado/__init__.py b/python/ql/test/query-tests/Security/lib/tornado/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/tornado/web.py b/python/ql/test/query-tests/Security/lib/tornado/web.py deleted file mode 100644 index 41b91848992..00000000000 --- a/python/ql/test/query-tests/Security/lib/tornado/web.py +++ /dev/null @@ -1,2 +0,0 @@ -class RequestHandler(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/traceback.py b/python/ql/test/query-tests/Security/lib/traceback.py deleted file mode 100644 index 2a7c5e58847..00000000000 --- a/python/ql/test/query-tests/Security/lib/traceback.py +++ /dev/null @@ -1,2 +0,0 @@ -def format_exc(): - return None \ No newline at end of file diff --git a/python/ql/test/query-tests/Security/lib/twisted/__init__.py b/python/ql/test/query-tests/Security/lib/twisted/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/twisted/web/__init__.py b/python/ql/test/query-tests/Security/lib/twisted/web/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/twisted/web/http.py b/python/ql/test/query-tests/Security/lib/twisted/web/http.py deleted file mode 100644 index c7ac65eaf05..00000000000 --- a/python/ql/test/query-tests/Security/lib/twisted/web/http.py +++ /dev/null @@ -1,2 +0,0 @@ -class Request(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/twisted/web/resource.py b/python/ql/test/query-tests/Security/lib/twisted/web/resource.py deleted file mode 100644 index 1fe186864cb..00000000000 --- a/python/ql/test/query-tests/Security/lib/twisted/web/resource.py +++ /dev/null @@ -1,2 +0,0 @@ -class Resource(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/yaml.py b/python/ql/test/query-tests/Security/lib/yaml.py deleted file mode 100644 index 51fe2fd1d42..00000000000 --- a/python/ql/test/query-tests/Security/lib/yaml.py +++ /dev/null @@ -1,2 +0,0 @@ -def load(s): - pass diff --git a/python/ql/test/query-tests/Security/lib/zope/__init__.py b/python/ql/test/query-tests/Security/lib/zope/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/zope/interface/__init__.py b/python/ql/test/query-tests/Security/lib/zope/interface/__init__.py deleted file mode 100644 index 423c7b58341..00000000000 --- a/python/ql/test/query-tests/Security/lib/zope/interface/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -class Interface(): - pass - - -class implementer: - - def __call__(self, ob): - ... - return ob From ff498f616a9896fb18f693d2cb4c36cd9be4f8d7 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Wed, 10 Apr 2024 13:20:57 +0200 Subject: [PATCH 48/50] C#: Some renaming. --- .../DependencyManager.Nuget.cs | 14 ++--- .../DependencyManager.cs | 61 +++++++++---------- 2 files changed, 37 insertions(+), 38 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs index 40f240f3145..cf9c7253b17 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs @@ -12,14 +12,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { public sealed partial class DependencyManager { - private void RestoreNugetPackages(List allNonBinaryFiles, IEnumerable allProjects, IEnumerable allSolutions, HashSet dllPaths) + private void RestoreNugetPackages(List allNonBinaryFiles, IEnumerable allProjects, IEnumerable allSolutions, HashSet dllLocations) { try { var checkNugetFeedResponsiveness = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.CheckNugetFeedResponsiveness); if (checkNugetFeedResponsiveness && !CheckFeeds(allNonBinaryFiles)) { - DownloadMissingPackages(allNonBinaryFiles, dllPaths, withNugetConfig: false); + DownloadMissingPackages(allNonBinaryFiles, dllLocations, withNugetConfig: false); return; } @@ -55,7 +55,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } nugetPackageDllPaths.ExceptWith(excludedPaths); - dllPaths.UnionWith(nugetPackageDllPaths.Select(p => new AssemblyLookupLocation(p))); + dllLocations.UnionWith(nugetPackageDllPaths.Select(p => new AssemblyLookupLocation(p))); } catch (Exception exc) { @@ -72,10 +72,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching .Paths .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d)) .ToList(); - dllPaths.UnionWith(paths.Select(p => new AssemblyLookupLocation(p))); + dllLocations.UnionWith(paths.Select(p => new AssemblyLookupLocation(p))); LogAllUnusedPackages(dependencies); - DownloadMissingPackages(allNonBinaryFiles, dllPaths); + DownloadMissingPackages(allNonBinaryFiles, dllLocations); } /// @@ -148,7 +148,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString())); } - private void DownloadMissingPackages(List allFiles, ISet dllPaths, bool withNugetConfig = true) + private void DownloadMissingPackages(List allFiles, ISet dllLocations, bool withNugetConfig = true) { var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo); var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames(); @@ -206,7 +206,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString())); - dllPaths.Add(missingPackageDirectory.DirInfo.FullName); + dllLocations.Add(missingPackageDirectory.DirInfo.FullName); } private string[] GetAllNugetConfigs(List allFiles) => allFiles.SelectFileNamesByName("nuget.config").ToArray(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index 8115b0e8ae5..7f8306c2d9d 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -5,7 +5,6 @@ using System.IO; using System.Linq; using System.Security.Cryptography; using System.Text; -using System.Text.RegularExpressions; using System.Threading.Tasks; using Semmle.Util; @@ -91,9 +90,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching this.generatedSources = new(); var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj").ToList(); var allSolutions = allNonBinaryFiles.SelectFileNamesByExtension(".sln").ToList(); - var dllPaths = allFiles.SelectFileNamesByExtension(".dll").Select(x => new AssemblyLookupLocation(x)).ToHashSet(); + var dllLocations = allFiles.SelectFileNamesByExtension(".dll").Select(x => new AssemblyLookupLocation(x)).ToHashSet(); - logger.LogInfo($"Found {allFiles.Count} files, {nonGeneratedSources.Count} source files, {allProjects.Count} project files, {allSolutions.Count} solution files, {dllPaths.Count} DLLs."); + logger.LogInfo($"Found {allFiles.Count} files, {nonGeneratedSources.Count} source files, {allProjects.Count} project files, {allSolutions.Count} solution files, {dllLocations.Count} DLLs."); void startCallback(string s, bool silent) { @@ -122,12 +121,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching throw; } - RestoreNugetPackages(allNonBinaryFiles, allProjects, allSolutions, dllPaths); + RestoreNugetPackages(allNonBinaryFiles, allProjects, allSolutions, dllLocations); // Find DLLs in the .Net / Asp.Net Framework // This needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies. - var frameworkLocations = AddFrameworkDlls(dllPaths); + var frameworkLocations = AddFrameworkDlls(dllLocations); - assemblyCache = new AssemblyCache(dllPaths, frameworkLocations, logger); + assemblyCache = new AssemblyCache(dllLocations, frameworkLocations, logger); AnalyseSolutions(allSolutions); foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename)) @@ -192,7 +191,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching ]); } - private HashSet AddFrameworkDlls(HashSet dllPaths) + private HashSet AddFrameworkDlls(HashSet dllLocations) { var frameworkLocations = new HashSet(); @@ -200,9 +199,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var useSubfolders = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders); if (!string.IsNullOrWhiteSpace(frameworkReferences)) { - RemoveFrameworkNugetPackages(dllPaths); - RemoveNugetPackageReference(FrameworkPackageNames.AspNetCoreFramework, dllPaths); - RemoveNugetPackageReference(FrameworkPackageNames.WindowsDesktopFramework, dllPaths); + RemoveFrameworkNugetPackages(dllLocations); + RemoveNugetPackageReference(FrameworkPackageNames.AspNetCoreFramework, dllLocations); + RemoveNugetPackageReference(FrameworkPackageNames.WindowsDesktopFramework, dllLocations); var frameworkPaths = frameworkReferences.Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries); @@ -214,16 +213,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching continue; } - dllPaths.Add(new AssemblyLookupLocation(path, _ => true, useSubfolders)); + dllLocations.Add(new AssemblyLookupLocation(path, _ => true, useSubfolders)); frameworkLocations.Add(path); } return frameworkLocations; } - AddNetFrameworkDlls(dllPaths, frameworkLocations); - AddAspNetCoreFrameworkDlls(dllPaths, frameworkLocations); - AddMicrosoftWindowsDesktopDlls(dllPaths, frameworkLocations); + AddNetFrameworkDlls(dllLocations, frameworkLocations); + AddAspNetCoreFrameworkDlls(dllLocations, frameworkLocations); + AddMicrosoftWindowsDesktopDlls(dllLocations, frameworkLocations); return frameworkLocations; } @@ -263,7 +262,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet dllPaths, ISet frameworkLocations) + private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet dllLocations, ISet frameworkLocations) { var versionFolders = GetPackageVersionSubDirectories(frameworkPath); if (versionFolders.Length > 1) @@ -279,7 +278,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching selectedFrameworkFolder = frameworkPath; } - dllPaths.Add(selectedFrameworkFolder); + dllLocations.Add(selectedFrameworkFolder); frameworkLocations.Add(selectedFrameworkFolder); logger.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {selectedFrameworkFolder}."); } @@ -292,16 +291,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching .ToArray(); } - private void RemoveFrameworkNugetPackages(ISet dllPaths, int fromIndex = 0) + private void RemoveFrameworkNugetPackages(ISet dllLocations, int fromIndex = 0) { var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks; for (var i = fromIndex; i < packagesInPrioOrder.Length; i++) { - RemoveNugetPackageReference(packagesInPrioOrder[i], dllPaths); + RemoveNugetPackageReference(packagesInPrioOrder[i], dllLocations); } } - private void AddNetFrameworkDlls(ISet dllPaths, ISet frameworkLocations) + private void AddNetFrameworkDlls(ISet dllLocations, ISet frameworkLocations) { // Multiple dotnet framework packages could be present. // The order of the packages is important, we're adding the first one that is present in the nuget cache. @@ -321,8 +320,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching dotnetFrameworkVersionVariantCount += GetPackageVersionSubDirectories(fp.Path!).Length; } - SelectNewestFrameworkPath(frameworkPath.Path, ".NET Framework", dllPaths, frameworkLocations); - RemoveFrameworkNugetPackages(dllPaths, frameworkPath.Index + 1); + SelectNewestFrameworkPath(frameworkPath.Path, ".NET Framework", dllLocations, frameworkLocations); + RemoveFrameworkNugetPackages(dllLocations, frameworkPath.Index + 1); return; } @@ -350,18 +349,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (runtimeLocation is null) { runtimeLocation ??= Runtime.ExecutingRuntime; - dllPaths.Add(new AssemblyLookupLocation(runtimeLocation, name => !name.StartsWith("Semmle."))); + dllLocations.Add(new AssemblyLookupLocation(runtimeLocation, name => !name.StartsWith("Semmle."))); } else { - dllPaths.Add(runtimeLocation); + dllLocations.Add(runtimeLocation); } logger.LogInfo($".NET runtime location selected: {runtimeLocation}"); frameworkLocations.Add(runtimeLocation); } - private void RemoveNugetPackageReference(string packagePrefix, ISet dllPaths) + private void RemoveNugetPackageReference(string packagePrefix, ISet dllLocations) { var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant(); if (packageFolder == null) @@ -370,10 +369,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } var packagePathPrefix = Path.Combine(packageFolder, packagePrefix.ToLowerInvariant()); - var toRemove = dllPaths.Where(s => s.Path.StartsWith(packagePathPrefix, StringComparison.InvariantCultureIgnoreCase)); + var toRemove = dllLocations.Where(s => s.Path.StartsWith(packagePathPrefix, StringComparison.InvariantCultureIgnoreCase)); foreach (var path in toRemove) { - dllPaths.Remove(path); + dllLocations.Remove(path); logger.LogInfo($"Removed reference {path}"); } } @@ -383,7 +382,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return fileContent.IsNewProjectStructureUsed && fileContent.UseAspNetCoreDlls; } - private void AddAspNetCoreFrameworkDlls(ISet dllPaths, ISet frameworkLocations) + private void AddAspNetCoreFrameworkDlls(ISet dllLocations, ISet frameworkLocations) { if (!IsAspNetCoreDetected()) { @@ -393,23 +392,23 @@ namespace Semmle.Extraction.CSharp.DependencyFetching // First try to find ASP.NET Core assemblies in the NuGet packages if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework, packageDirectory) is string aspNetCorePackage) { - SelectNewestFrameworkPath(aspNetCorePackage, "ASP.NET Core", dllPaths, frameworkLocations); + SelectNewestFrameworkPath(aspNetCorePackage, "ASP.NET Core", dllLocations, frameworkLocations); return; } if (Runtime.AspNetCoreRuntime is string aspNetCoreRuntime) { logger.LogInfo($"ASP.NET runtime location selected: {aspNetCoreRuntime}"); - dllPaths.Add(aspNetCoreRuntime); + dllLocations.Add(aspNetCoreRuntime); frameworkLocations.Add(aspNetCoreRuntime); } } - private void AddMicrosoftWindowsDesktopDlls(ISet dllPaths, ISet frameworkLocations) + private void AddMicrosoftWindowsDesktopDlls(ISet dllLocations, ISet frameworkLocations) { if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework, packageDirectory) is string windowsDesktopApp) { - SelectNewestFrameworkPath(windowsDesktopApp, "Windows Desktop App", dllPaths, frameworkLocations); + SelectNewestFrameworkPath(windowsDesktopApp, "Windows Desktop App", dllLocations, frameworkLocations); } } From 4ae25c2d34d7b3c3c72a47a3ea14d2b7d7f7a912 Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Wed, 10 Apr 2024 14:26:00 +0200 Subject: [PATCH 49/50] don't mention arrays in the qhelp for rb/shell-command-constructed-from-input, because there are no array --- .../security/cwe-078/UnsafeShellCommandConstruction.qhelp | 7 ++++++- .../examples/unsafe-shell-command-construction_fixed.rb | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ruby/ql/src/queries/security/cwe-078/UnsafeShellCommandConstruction.qhelp b/ruby/ql/src/queries/security/cwe-078/UnsafeShellCommandConstruction.qhelp index 4231f7cb0b4..8cedb57c277 100644 --- a/ruby/ql/src/queries/security/cwe-078/UnsafeShellCommandConstruction.qhelp +++ b/ruby/ql/src/queries/security/cwe-078/UnsafeShellCommandConstruction.qhelp @@ -20,10 +20,15 @@

- If possible, provide the dynamic arguments to the shell as an array + If possible, avoid concatenating shell strings to APIs such as system(..) to avoid interpretation by the shell.

+

+ Instead, provide the arguments to the shell command as separate arguments to the + API, such as system("echo", arg1, arg2). +

+

Alternatively, if the shell command must be constructed dynamically, then add code to ensure that special characters diff --git a/ruby/ql/src/queries/security/cwe-078/examples/unsafe-shell-command-construction_fixed.rb b/ruby/ql/src/queries/security/cwe-078/examples/unsafe-shell-command-construction_fixed.rb index cb8730eee09..b055f94eba9 100644 --- a/ruby/ql/src/queries/security/cwe-078/examples/unsafe-shell-command-construction_fixed.rb +++ b/ruby/ql/src/queries/security/cwe-078/examples/unsafe-shell-command-construction_fixed.rb @@ -1,6 +1,6 @@ module Utils def download(path) - # using an array to call `system` is safe + # using an API that doesn't interpret the path as a shell command system("wget", path) # OK end end \ No newline at end of file From ef68e3344980006eec8eabef044053d3b69d7452 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Wed, 10 Apr 2024 14:48:02 +0200 Subject: [PATCH 50/50] C#: Address review comments. --- .../AssemblyLookupLocation.cs | 37 ++++++++++--------- .../global.json | 5 --- 2 files changed, 19 insertions(+), 23 deletions(-) delete mode 100644 csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/global.json diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs index 533b547e45d..98f43b9dcb7 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs @@ -10,11 +10,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// Used to represent a path to an assembly or a directory containing assemblies /// and a selector function to determine which files to include, when indexing the assemblies. ///

- internal sealed class AssemblyLookupLocation(string p, Func includeFileName, bool indexSubdirectories = true) + internal sealed class AssemblyLookupLocation(string path, Func includeFileName, bool indexSubdirectories = true) { - public string Path => p; + public string Path => path; - public AssemblyLookupLocation(string p) : this(p, _ => true) { } + public AssemblyLookupLocation(string path) : this(path, _ => true) { } public static implicit operator AssemblyLookupLocation(string path) => new(path); @@ -23,15 +23,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// and adds them to the a list of assembly names to index. /// Indexing is performed at a later stage. This only collects the names. /// - /// The directory to index. + /// List of dlls to index. + /// Logger. private void AddReferenceDirectory(List dllsToIndex, ILogger logger) { try { - var dlls = new DirectoryInfo(p).EnumerateFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = indexSubdirectories, MatchCasing = MatchCasing.CaseInsensitive, AttributesToSkip = FileAttributes.None }); + var dlls = new DirectoryInfo(path).EnumerateFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = indexSubdirectories, MatchCasing = MatchCasing.CaseInsensitive, AttributesToSkip = FileAttributes.None }); if (!dlls.Any()) { - logger.LogWarning($"AssemblyLookupLocation: No DLLs found in the path '{p}'."); + logger.LogWarning($"AssemblyLookupLocation: No DLLs found in the path '{path}'."); return; } foreach (var dll in dlls) @@ -48,47 +49,47 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } catch (Exception e) { - logger.LogError($"AssemblyLookupLocation: Error while searching for DLLs in '{p}': {e.Message}"); + logger.LogError($"AssemblyLookupLocation: Error while searching for DLLs in '{path}': {e.Message}"); } } /// - /// Returns a list of paths to all assemblies in `p` that should be indexed. + /// Returns a list of paths to all assemblies in `path` that should be indexed. /// /// Logger public List GetDlls(ILogger logger) { var dllsToIndex = new List(); - if (File.Exists(p)) + if (File.Exists(path)) { - if (includeFileName(System.IO.Path.GetFileName(p))) + if (includeFileName(System.IO.Path.GetFileName(path))) { - dllsToIndex.Add(p); + dllsToIndex.Add(path); } else { - logger.LogInfo($"AssemblyLookupLocation: Skipping {p}."); + logger.LogInfo($"AssemblyLookupLocation: Skipping {path}."); } return dllsToIndex; } - if (Directory.Exists(p)) + if (Directory.Exists(path)) { - logger.LogInfo($"AssemblyLookupLocation: Finding reference DLLs in {p}..."); + logger.LogInfo($"AssemblyLookupLocation: Finding reference DLLs in {path}..."); AddReferenceDirectory(dllsToIndex, logger); } else { - logger.LogInfo("AssemblyLookupLocation: Path not found: " + p); + logger.LogInfo("AssemblyLookupLocation: Path not found: " + path); } return dllsToIndex; } public override bool Equals(object? obj) => - obj is AssemblyLookupLocation ap && p.Equals(ap.Path); + obj is AssemblyLookupLocation ap && path.Equals(ap.Path); - public override int GetHashCode() => p.GetHashCode(); + public override int GetHashCode() => path.GetHashCode(); - public override string ToString() => p; + public override string ToString() => path; } } diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/global.json b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/global.json deleted file mode 100644 index d54915e8d4d..00000000000 --- a/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/global.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "sdk": { - "version": "8.0.101" - } -}