From 0515b12305c844e0ce1d5dcd8e03eb62e4350b5c Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Mon, 25 Mar 2024 11:36:21 +0100 Subject: [PATCH 01/39] JS: Add example of bad NodeJS detection Notice the TRAP lines ``` is_module(#20001) is_es2015_module(#20001) ``` --- .../extractor/tests/node/input/detection.js | 6 + .../tests/node/output/trap/detection.js.trap | 203 ++++++++++++++++++ 2 files changed, 209 insertions(+) create mode 100644 javascript/extractor/tests/node/input/detection.js create mode 100644 javascript/extractor/tests/node/output/trap/detection.js.trap diff --git a/javascript/extractor/tests/node/input/detection.js b/javascript/extractor/tests/node/input/detection.js new file mode 100644 index 00000000000..7c182f394f2 --- /dev/null +++ b/javascript/extractor/tests/node/input/detection.js @@ -0,0 +1,6 @@ +// the comment below (with 'import' on line starting with whitespace) caused the +// extractor to think it was a es2015 module and not a commonjs module. +/* + import +*/ +const fs = require('fs'); diff --git a/javascript/extractor/tests/node/output/trap/detection.js.trap b/javascript/extractor/tests/node/output/trap/detection.js.trap new file mode 100644 index 00000000000..1da6b26f89c --- /dev/null +++ b/javascript/extractor/tests/node/output/trap/detection.js.trap @@ -0,0 +1,203 @@ +#10000=@"/detection.js;sourcefile" +files(#10000,"/detection.js") +#10001=@"/;folder" +folders(#10001,"/") +containerparent(#10001,#10000) +#10002=@"loc,{#10000},0,0,0,0" +locations_default(#10002,#10000,0,0,0,0) +hasLocation(#10000,#10002) +#20000=@"global_scope" +scopes(#20000,0) +#20001=@"script;{#10000},1,1" +#20002=* +comments(#20002,0,#20001," the comment below (with 'import' on line starting with whitespace) caused the","// the ... sed the") +#20003=@"loc,{#10000},1,1,1,80" +locations_default(#20003,#10000,1,1,1,80) +hasLocation(#20002,#20003) +#20004=* +comments(#20004,0,#20001," extractor to think it was a es2015 module and not a commonjs module.","// extr ... module.") +#20005=@"loc,{#10000},2,1,2,71" +locations_default(#20005,#10000,2,1,2,71) +hasLocation(#20004,#20005) +#20006=* +comments(#20006,1,#20001," + import +","/*\n import\n*/") +#20007=@"loc,{#10000},3,1,5,2" +locations_default(#20007,#10000,3,1,5,2) +hasLocation(#20006,#20007) +#20008=* +lines(#20008,#20001,"// the comment below (with 'import' on line starting with whitespace) caused the"," +") +hasLocation(#20008,#20003) +#20009=* +lines(#20009,#20001,"// extractor to think it was a es2015 module and not a commonjs module."," +") +hasLocation(#20009,#20005) +#20010=* +lines(#20010,#20001,"/*"," +") +#20011=@"loc,{#10000},3,1,3,2" +locations_default(#20011,#10000,3,1,3,2) +hasLocation(#20010,#20011) +#20012=* +lines(#20012,#20001," import"," +") +#20013=@"loc,{#10000},4,1,4,8" +locations_default(#20013,#10000,4,1,4,8) +hasLocation(#20012,#20013) +indentation(#10000,4," ",2) +#20014=* +lines(#20014,#20001,"*/"," +") +#20015=@"loc,{#10000},5,1,5,2" +locations_default(#20015,#10000,5,1,5,2) +hasLocation(#20014,#20015) +#20016=* +lines(#20016,#20001,"const fs = require('fs');"," +") +#20017=@"loc,{#10000},6,1,6,25" +locations_default(#20017,#10000,6,1,6,25) +hasLocation(#20016,#20017) +numlines(#20001,6,1,5) +#20018=* +tokeninfo(#20018,7,#20001,0,"const") +#20019=@"loc,{#10000},6,1,6,5" +locations_default(#20019,#10000,6,1,6,5) +hasLocation(#20018,#20019) +next_token(#20002,#20018) +next_token(#20004,#20018) +next_token(#20006,#20018) +#20020=* +tokeninfo(#20020,6,#20001,1,"fs") +#20021=@"loc,{#10000},6,7,6,8" +locations_default(#20021,#10000,6,7,6,8) +hasLocation(#20020,#20021) +#20022=* +tokeninfo(#20022,8,#20001,2,"=") +#20023=@"loc,{#10000},6,10,6,10" +locations_default(#20023,#10000,6,10,6,10) +hasLocation(#20022,#20023) +#20024=* +tokeninfo(#20024,6,#20001,3,"require") +#20025=@"loc,{#10000},6,12,6,18" +locations_default(#20025,#10000,6,12,6,18) +hasLocation(#20024,#20025) +#20026=* +tokeninfo(#20026,8,#20001,4,"(") +#20027=@"loc,{#10000},6,19,6,19" +locations_default(#20027,#10000,6,19,6,19) +hasLocation(#20026,#20027) +#20028=* +tokeninfo(#20028,4,#20001,5,"'fs'") +#20029=@"loc,{#10000},6,20,6,23" +locations_default(#20029,#10000,6,20,6,23) +hasLocation(#20028,#20029) +#20030=* +tokeninfo(#20030,8,#20001,6,")") +#20031=@"loc,{#10000},6,24,6,24" +locations_default(#20031,#10000,6,24,6,24) +hasLocation(#20030,#20031) +#20032=* +tokeninfo(#20032,8,#20001,7,";") +#20033=@"loc,{#10000},6,25,6,25" +locations_default(#20033,#10000,6,25,6,25) +hasLocation(#20032,#20033) +#20034=* +tokeninfo(#20034,0,#20001,8,"") +#20035=@"loc,{#10000},7,1,7,0" +locations_default(#20035,#10000,7,1,7,0) +hasLocation(#20034,#20035) +toplevels(#20001,0) +#20036=@"loc,{#10000},1,1,7,0" +locations_default(#20036,#10000,1,1,7,0) +hasLocation(#20001,#20036) +#20037=@"var;{global};{#20000}" +variables(#20037,"global",#20000) +#20038=@"var;{process};{#20000}" +variables(#20038,"process",#20000) +#20039=@"var;{console};{#20000}" +variables(#20039,"console",#20000) +#20040=@"var;{Buffer};{#20000}" +variables(#20040,"Buffer",#20000) +#20041=@"module;{#10000},1,1" +scopes(#20041,3) +scopenodes(#20001,#20041) +scopenesting(#20041,#20000) +#20042=@"var;{require};{#20041}" +variables(#20042,"require",#20041) +#20043=@"var;{module};{#20041}" +variables(#20043,"module",#20041) +#20044=@"var;{exports};{#20041}" +variables(#20044,"exports",#20041) +#20045=@"var;{__filename};{#20041}" +variables(#20045,"__filename",#20041) +#20046=@"var;{__dirname};{#20041}" +variables(#20046,"__dirname",#20041) +#20047=@"var;{arguments};{#20041}" +variables(#20047,"arguments",#20041) +is_module(#20001) +is_es2015_module(#20001) +#20048=@"var;{fs};{#20041}" +variables(#20048,"fs",#20041) +#20049=* +stmts(#20049,22,#20001,0,"const f ... ('fs');") +hasLocation(#20049,#20017) +stmt_containers(#20049,#20001) +#20050=* +exprs(#20050,64,#20049,0,"fs = require('fs')") +#20051=@"loc,{#10000},6,7,6,24" +locations_default(#20051,#10000,6,7,6,24) +hasLocation(#20050,#20051) +enclosing_stmt(#20050,#20049) +expr_containers(#20050,#20001) +#20052=* +exprs(#20052,78,#20050,0,"fs") +hasLocation(#20052,#20021) +enclosing_stmt(#20052,#20049) +expr_containers(#20052,#20001) +literals("fs","fs",#20052) +decl(#20052,#20048) +#20053=* +exprs(#20053,13,#20050,1,"require('fs')") +#20054=@"loc,{#10000},6,12,6,24" +locations_default(#20054,#10000,6,12,6,24) +hasLocation(#20053,#20054) +enclosing_stmt(#20053,#20049) +expr_containers(#20053,#20001) +#20055=* +exprs(#20055,79,#20053,-1,"require") +hasLocation(#20055,#20025) +enclosing_stmt(#20055,#20049) +expr_containers(#20055,#20001) +literals("require","require",#20055) +bind(#20055,#20042) +#20056=* +exprs(#20056,4,#20053,0,"'fs'") +hasLocation(#20056,#20029) +enclosing_stmt(#20056,#20049) +expr_containers(#20056,#20001) +literals("fs","'fs'",#20056) +#20057=* +regexpterm(#20057,14,#20056,0,"fs") +#20058=@"loc,{#10000},6,21,6,22" +locations_default(#20058,#10000,6,21,6,22) +hasLocation(#20057,#20058) +regexp_const_value(#20057,"fs") +#20059=* +entry_cfg_node(#20059,#20001) +#20060=@"loc,{#10000},1,1,1,0" +locations_default(#20060,#10000,1,1,1,0) +hasLocation(#20059,#20060) +#20061=* +exit_cfg_node(#20061,#20001) +hasLocation(#20061,#20035) +successor(#20049,#20052) +successor(#20056,#20053) +successor(#20055,#20056) +successor(#20053,#20050) +successor(#20052,#20055) +successor(#20050,#20061) +successor(#20059,#20049) +numlines(#10000,6,1,5) +filetype(#10000,"javascript") From cd84500c567e5c7b4cb4cfd7672f5270813cdd3e Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 26 Mar 2024 16:55:14 +0100 Subject: [PATCH 02/39] JS: Extractor: Separate base detector logic into own file Should hopefully make it easier to review these changes to have it split into its' own commit :) --- .../semmle/js/extractor/AbstractDetector.java | 113 ++++++++++++++++ .../semmle/js/extractor/NodeJSDetector.java | 124 +----------------- 2 files changed, 120 insertions(+), 117 deletions(-) create mode 100644 javascript/extractor/src/com/semmle/js/extractor/AbstractDetector.java diff --git a/javascript/extractor/src/com/semmle/js/extractor/AbstractDetector.java b/javascript/extractor/src/com/semmle/js/extractor/AbstractDetector.java new file mode 100644 index 00000000000..b8a5f1df0fc --- /dev/null +++ b/javascript/extractor/src/com/semmle/js/extractor/AbstractDetector.java @@ -0,0 +1,113 @@ +package com.semmle.js.extractor; + +import com.semmle.js.ast.AssignmentExpression; +import com.semmle.js.ast.BlockStatement; +import com.semmle.js.ast.CallExpression; +import com.semmle.js.ast.Expression; +import com.semmle.js.ast.ExpressionStatement; +import com.semmle.js.ast.IFunction; +import com.semmle.js.ast.Identifier; +import com.semmle.js.ast.IfStatement; +import com.semmle.js.ast.MemberExpression; +import com.semmle.js.ast.Node; +import com.semmle.js.ast.ParenthesizedExpression; +import com.semmle.js.ast.Program; +import com.semmle.js.ast.Statement; +import com.semmle.js.ast.TryStatement; +import com.semmle.js.ast.UnaryExpression; +import com.semmle.js.ast.VariableDeclaration; +import com.semmle.js.ast.VariableDeclarator; +import java.util.List; + +/** + * A utility base class for running detection logic on statements/expressions by + * visiting each node. It performs recursive decent into assignment expressions and + * callee expressions for call-expressions (to handle detection of `foo` in `foo()()`) + * */ +public abstract class AbstractDetector { + protected boolean programDetection(Node ast) { + if (!(ast instanceof Program)) return false; + + return visitStatements(((Program) ast).getBody()); + } + + protected boolean visitStatements(List stmts) { + for (Statement stmt : stmts) if (visitStatement(stmt)) return true; + return false; + } + + protected boolean visitStatement(Statement stmt) { + if (stmt instanceof ExpressionStatement) { + Expression e = stripParens(((ExpressionStatement) stmt).getExpression()); + + // check whether `e` is an iife; if so, recursively check its body + + // strip off unary operators to handle `!function(){...}()` + if (e instanceof UnaryExpression) e = ((UnaryExpression) e).getArgument(); + + if (e instanceof CallExpression && ((CallExpression) e).getArguments().isEmpty()) { + Expression callee = stripParens(((CallExpression) e).getCallee()); + if (callee instanceof IFunction) { + Node body = ((IFunction) callee).getBody(); + if (body instanceof BlockStatement) + return visitStatements(((BlockStatement) body).getBody()); + } + } + + if (visitExpression(e)) return true; + + } else if (stmt instanceof VariableDeclaration) { + for (VariableDeclarator decl : ((VariableDeclaration) stmt).getDeclarations()) { + Expression init = stripParens(decl.getInit()); + if (visitExpression(init)) return true; + } + + } else if (stmt instanceof TryStatement) { + return visitStatement(((TryStatement) stmt).getBlock()); + + } else if (stmt instanceof BlockStatement) { + return visitStatements(((BlockStatement) stmt).getBody()); + + } else if (stmt instanceof IfStatement) { + IfStatement is = (IfStatement) stmt; + return visitStatement(is.getConsequent()) + || visitStatement(is.getAlternate()); + } + + return false; + } + + private static Expression stripParens(Expression e) { + if (e instanceof ParenthesizedExpression) + return stripParens(((ParenthesizedExpression) e).getExpression()); + return e; + } + + /** + * Recursively check {@code e} if it's a call or an assignment. + */ + protected boolean visitExpression(Expression e) { + if (e instanceof CallExpression) { + CallExpression call = (CallExpression) e; + Expression callee = call.getCallee(); + // recurse, to handle things like `foo()()` + if (visitExpression(callee)) return true; + return false; + } else if (e instanceof MemberExpression) { + return visitExpression(((MemberExpression) e).getObject()); + } else if (e instanceof AssignmentExpression) { + AssignmentExpression assgn = (AssignmentExpression) e; + + // filter out compound assignments + if (!"=".equals(assgn.getOperator())) return false; + + return visitExpression(assgn.getRight()); + } + return false; + } + + /** Is {@code e} an identifier with name {@code name}? */ + protected static boolean isIdentifier(Expression e, String name) { + return e instanceof Identifier && name.equals(((Identifier) e).getName()); + } +} diff --git a/javascript/extractor/src/com/semmle/js/extractor/NodeJSDetector.java b/javascript/extractor/src/com/semmle/js/extractor/NodeJSDetector.java index d848afd0d87..a9fa2c9d850 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/NodeJSDetector.java +++ b/javascript/extractor/src/com/semmle/js/extractor/NodeJSDetector.java @@ -1,132 +1,30 @@ package com.semmle.js.extractor; import com.semmle.js.ast.AssignmentExpression; -import com.semmle.js.ast.BlockStatement; import com.semmle.js.ast.CallExpression; import com.semmle.js.ast.Expression; -import com.semmle.js.ast.ExpressionStatement; -import com.semmle.js.ast.IFunction; -import com.semmle.js.ast.Identifier; -import com.semmle.js.ast.IfStatement; import com.semmle.js.ast.MemberExpression; import com.semmle.js.ast.Node; -import com.semmle.js.ast.ParenthesizedExpression; -import com.semmle.js.ast.Program; -import com.semmle.js.ast.Statement; -import com.semmle.js.ast.TryStatement; -import com.semmle.js.ast.UnaryExpression; -import com.semmle.js.ast.VariableDeclaration; -import com.semmle.js.ast.VariableDeclarator; -import java.util.List; /** A utility class for detecting Node.js code. */ -public class NodeJSDetector { +public class NodeJSDetector extends AbstractDetector { /** * Is {@code ast} a program that looks like Node.js code, that is, does it contain a top-level - * {@code require} or an export? + * {@code require} or an {@code module.exports = ...}/{@code exports = ...}? */ public static boolean looksLikeNodeJS(Node ast) { - if (!(ast instanceof Program)) return false; - - return hasToplevelRequireOrExport(((Program) ast).getBody()); + return new NodeJSDetector().programDetection(ast); } - /** - * Does this program contain a statement that looks like a Node.js {@code require} or an export? - * - *

We recursively traverse argument-less immediately invoked function expressions (i.e., no UMD - * modules), but not loops or if statements. - */ - private static boolean hasToplevelRequireOrExport(List stmts) { - for (Statement stmt : stmts) if (hasToplevelRequireOrExport(stmt)) return true; - return false; - } - - private static boolean hasToplevelRequireOrExport(Statement stmt) { - if (stmt instanceof ExpressionStatement) { - Expression e = stripParens(((ExpressionStatement) stmt).getExpression()); - - // check whether `e` is an iife; if so, recursively check its body - - // strip off unary operators to handle `!function(){...}()` - if (e instanceof UnaryExpression) e = ((UnaryExpression) e).getArgument(); - - if (e instanceof CallExpression && ((CallExpression) e).getArguments().isEmpty()) { - Expression callee = stripParens(((CallExpression) e).getCallee()); - if (callee instanceof IFunction) { - Node body = ((IFunction) callee).getBody(); - if (body instanceof BlockStatement) - return hasToplevelRequireOrExport(((BlockStatement) body).getBody()); - } - } - - if (isRequireCall(e) || isExport(e)) return true; - - } else if (stmt instanceof VariableDeclaration) { - for (VariableDeclarator decl : ((VariableDeclaration) stmt).getDeclarations()) { - Expression init = stripParens(decl.getInit()); - if (isRequireCall(init) || isExport(init)) return true; - } - - } else if (stmt instanceof TryStatement) { - return hasToplevelRequireOrExport(((TryStatement) stmt).getBlock()); - - } else if (stmt instanceof BlockStatement) { - return hasToplevelRequireOrExport(((BlockStatement) stmt).getBody()); - - } else if (stmt instanceof IfStatement) { - IfStatement is = (IfStatement) stmt; - return hasToplevelRequireOrExport(is.getConsequent()) - || hasToplevelRequireOrExport(is.getAlternate()); - } - - return false; - } - - private static Expression stripParens(Expression e) { - if (e instanceof ParenthesizedExpression) - return stripParens(((ParenthesizedExpression) e).getExpression()); - return e; - } - - /** - * Is {@code e} a call to a function named {@code require} with one argument, or an assignment - * whose right hand side is the result of such a call? - */ - private static boolean isRequireCall(Expression e) { + @Override + protected boolean visitExpression(Expression e) { + // require('...') if (e instanceof CallExpression) { CallExpression call = (CallExpression) e; Expression callee = call.getCallee(); if (isIdentifier(callee, "require") && call.getArguments().size() == 1) return true; - if (isRequireCall(callee)) return true; - return false; - } else if (e instanceof MemberExpression) { - return isRequireCall(((MemberExpression) e).getObject()); - } else if (e instanceof AssignmentExpression) { - AssignmentExpression assgn = (AssignmentExpression) e; - - // filter out compound assignments - if (!"=".equals(assgn.getOperator())) return false; - - return isRequireCall(assgn.getRight()); } - return false; - } - /** - * Does {@code e} look like a Node.js export? - * - *

Currently, three kinds of exports are recognised: - * - *

    - *
  • exports.foo = ... - *
  • module.exports = ... - *
  • module.exports.foo = ... - *
- * - * Detection is done recursively, so foo = exports.foo = ... is handled correctly. - */ - private static boolean isExport(Expression e) { if (e instanceof AssignmentExpression) { AssignmentExpression assgn = (AssignmentExpression) e; @@ -149,12 +47,9 @@ public class NodeJSDetector { if (isModuleExports(targetBase)) return true; } } - - // recursively check right hand side - return isExport(assgn.getRight()); } - return false; + return super.visitExpression(e); } /** Is {@code me} a member expression {@code module.exports}? */ @@ -163,9 +58,4 @@ public class NodeJSDetector { && isIdentifier(me.getObject(), "module") && isIdentifier(me.getProperty(), "exports"); } - - /** Is {@code e} an identifier with name {@code name}? */ - private static boolean isIdentifier(Expression e, String name) { - return e instanceof Identifier && name.equals(((Identifier) e).getName()); - } } From 04a0740ccb693782703430faa098d39ef9a15031 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Mon, 25 Mar 2024 13:17:15 +0100 Subject: [PATCH 03/39] JS: Extractor: More robust ES2015 checking Created shared AbstractDetector to not duplicate all the tedious logic ;) I took inspiration from the tests in `javascript/extractor/tests/esnext/input/dynamic-import.js` --- .../semmle/js/extractor/ES2015Detector.java | 34 +++++++++++ .../com/semmle/js/extractor/JSExtractor.java | 20 +++++++ .../semmle/js/extractor/test/AllTests.java | 1 + .../extractor/test/ES2015DetectorTests.java | 58 +++++++++++++++++++ .../tests/node/output/trap/detection.js.trap | 2 +- 5 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 javascript/extractor/src/com/semmle/js/extractor/ES2015Detector.java create mode 100644 javascript/extractor/test/com/semmle/js/extractor/test/ES2015DetectorTests.java diff --git a/javascript/extractor/src/com/semmle/js/extractor/ES2015Detector.java b/javascript/extractor/src/com/semmle/js/extractor/ES2015Detector.java new file mode 100644 index 00000000000..3ba63963dab --- /dev/null +++ b/javascript/extractor/src/com/semmle/js/extractor/ES2015Detector.java @@ -0,0 +1,34 @@ +package com.semmle.js.extractor; + +import com.semmle.js.ast.DynamicImport; +import com.semmle.js.ast.ExportDeclaration; +import com.semmle.js.ast.Expression; +import com.semmle.js.ast.ImportDeclaration; +import com.semmle.js.ast.Node; +import com.semmle.js.ast.Statement; + +/** A utility class for detecting Node.js code. */ +public class ES2015Detector extends AbstractDetector { + /** + * Is {@code ast} a program that uses ES2015 import/export code? + */ + public static boolean looksLikeES2015(Node ast) { + return new ES2015Detector().programDetection(ast); + } + + @Override + protected boolean visitStatement(Statement stmt) { + if (stmt instanceof ImportDeclaration || stmt instanceof ExportDeclaration) { + return true; + } + return super.visitStatement(stmt); + } + + @Override + protected boolean visitExpression(Expression e) { + if (e instanceof DynamicImport) { + return true; + } + return super.visitExpression(e); + } +} diff --git a/javascript/extractor/src/com/semmle/js/extractor/JSExtractor.java b/javascript/extractor/src/com/semmle/js/extractor/JSExtractor.java index 6b4b05fcf61..a8e20fdc0d9 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/JSExtractor.java +++ b/javascript/extractor/src/com/semmle/js/extractor/JSExtractor.java @@ -58,6 +58,26 @@ public class JSExtractor { JSParser.Result parserRes = JSParser.parse(config, sourceType, source, textualExtractor.getMetrics()); + + // Check if we guessed wrong with the regex in `establishSourceType`, (which could + // happen due to a block-comment line starting with ' import'). + if (config.getSourceType() == SourceType.AUTO && sourceType != SourceType.SCRIPT) { + boolean wrongGuess = false; + + if (sourceType == SourceType.MODULE) { + // check that we did see an import/export declaration + wrongGuess = ES2015Detector.looksLikeES2015(parserRes.getAST()) == false; + } else if (sourceType == SourceType.CLOSURE_MODULE ) { + // TODO + } + + if (wrongGuess) { + sourceType = SourceType.SCRIPT; + parserRes = + JSParser.parse(config, sourceType, source, textualExtractor.getMetrics()); + } + } + return extract(textualExtractor, source, toplevelKind, scopeManager, sourceType, parserRes); } diff --git a/javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java b/javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java index 7487e68efc0..6a51b2e69c4 100644 --- a/javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java +++ b/javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java @@ -18,6 +18,7 @@ import org.junit.runners.Suite.SuiteClasses; @SuiteClasses({ JSXTests.class, NodeJSDetectorTests.class, + ES2015DetectorTests.class, TrapTests.class, ObjectRestSpreadTests.class, ClassPropertiesTests.class, diff --git a/javascript/extractor/test/com/semmle/js/extractor/test/ES2015DetectorTests.java b/javascript/extractor/test/com/semmle/js/extractor/test/ES2015DetectorTests.java new file mode 100644 index 00000000000..669824c076f --- /dev/null +++ b/javascript/extractor/test/com/semmle/js/extractor/test/ES2015DetectorTests.java @@ -0,0 +1,58 @@ +package com.semmle.js.extractor.test; + +import com.semmle.js.ast.Node; +import com.semmle.js.extractor.ES2015Detector; +import com.semmle.js.extractor.ExtractionMetrics; +import com.semmle.js.extractor.ExtractorConfig; +import com.semmle.js.extractor.ExtractorConfig.SourceType; +import com.semmle.js.parser.JSParser; +import com.semmle.js.parser.JSParser.Result; +import org.junit.Assert; +import org.junit.Test; + +public class ES2015DetectorTests { + // using `experimental: true` as we do in real extractor, see `extractSource` method + // in `AutoBuild.java` + private static final ExtractorConfig CONFIG = new ExtractorConfig(true); + + private void isES2015(String src, boolean expected) { + Result res = JSParser.parse(CONFIG, SourceType.MODULE, src, new ExtractionMetrics()); + Node ast = res.getAST(); + Assert.assertNotNull(ast); + Assert.assertTrue(ES2015Detector.looksLikeES2015(ast) == expected); + } + + @Test + public void testImport() { + isES2015("import * as fs from 'fs';", true); + } + + @Test + public void testExport() { + isES2015("export function foo() { };", true); + } + + @Test + public void testDynamicImport() { + isES2015("import('fs');", true); + } + + @Test + public void testDynamicImportAssign() { + isES2015("var fs = import('fs');", true); + } + + @Test + public void testDynamicImportThen() { + isES2015("import('o').then((o) => {});", true); + } + + @Test + public void importInBlockComment() { + isES2015("/*\n" + + " import * from 'fs';\n" + + "*/\n" + + "const fs = require('fs');", + false); + } +} diff --git a/javascript/extractor/tests/node/output/trap/detection.js.trap b/javascript/extractor/tests/node/output/trap/detection.js.trap index 1da6b26f89c..81e56c20609 100644 --- a/javascript/extractor/tests/node/output/trap/detection.js.trap +++ b/javascript/extractor/tests/node/output/trap/detection.js.trap @@ -137,7 +137,6 @@ variables(#20046,"__dirname",#20041) #20047=@"var;{arguments};{#20041}" variables(#20047,"arguments",#20041) is_module(#20001) -is_es2015_module(#20001) #20048=@"var;{fs};{#20041}" variables(#20048,"fs",#20041) #20049=* @@ -199,5 +198,6 @@ successor(#20053,#20050) successor(#20052,#20055) successor(#20050,#20061) successor(#20059,#20049) +is_nodejs(#20001) numlines(#10000,6,1,5) filetype(#10000,"javascript") From 1d51d182ec4fba8c3c21aca83b2e8434f2828f69 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 26 Mar 2024 13:28:38 +0100 Subject: [PATCH 04/39] JS: Extractor: Explain how to make `replaceExpectedOutput` work now with bazel --- .../test/com/semmle/js/extractor/test/BUILD.bazel | 3 +++ .../test/com/semmle/js/extractor/test/TrapTests.java | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel b/javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel index 45d6790e8a9..1bad97bb7cc 100644 --- a/javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel +++ b/javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel @@ -12,6 +12,9 @@ java_test( "TS_WRAPPER_ZIP": "$(rlocationpath //javascript/extractor/lib/typescript)", }, test_class = "com.semmle.js.extractor.test.AllTests", + # To use `replaceExpectedOutput` you need to uncomment the following line + # (to be allowed to override the .trap files on disk) + # tags = ["no-sandbox"], deps = [ "//javascript/extractor", "//javascript/extractor:deps", diff --git a/javascript/extractor/test/com/semmle/js/extractor/test/TrapTests.java b/javascript/extractor/test/com/semmle/js/extractor/test/TrapTests.java index 5991ac6669a..6daf56cc56e 100644 --- a/javascript/extractor/test/com/semmle/js/extractor/test/TrapTests.java +++ b/javascript/extractor/test/com/semmle/js/extractor/test/TrapTests.java @@ -161,6 +161,14 @@ public class TrapTests { byte[] actual_utf8_bytes = StringUtil.stringToBytes(sw.toString()); String actual = new String(actual_utf8_bytes, Charset.forName("UTF-8")); File trap = new File(outputDir, f.getName() + ".trap"); + // NOTE: If you want to replace expected output, you MUST change + // the way this test is run under bazel to escape the bazel + // sandbox. Add `tags = ["no-sandbox"]` to the test rule in + // javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel + // + // if you have problems with too much caching, you need to find the right bazel command, + // and run `./build --bazel test ... --cache_test_results=no` + // (at least I had problems getting the "no-cache" tag to work) boolean replaceExpectedOutput = false; if (replaceExpectedOutput) { System.out.println("Replacing expected output for " + trap); From 60944a9bcbec317ed426fe50994fad1909ba614e Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 26 Mar 2024 13:29:45 +0100 Subject: [PATCH 05/39] JS: Accept new trap files As I see it, these all seem to have invalid code initially anyway, but this is definitely something a JS expert should review :) --- .../trap/anonFunctionWithoutParens.js.trap | 24 +-- .../flow/output/trap/anonIndexer.js.trap | 24 +-- .../trap/declared-module-imports.js.trap | 24 +-- .../flow/output/trap/exportOpaqueType.js.trap | 24 +-- .../trap/importTypeInDeclaredModule.js.trap | 24 +-- .../tests/flow/output/trap/variance.js.trap | 158 +++++++++--------- 6 files changed, 121 insertions(+), 157 deletions(-) diff --git a/javascript/extractor/tests/flow/output/trap/anonFunctionWithoutParens.js.trap b/javascript/extractor/tests/flow/output/trap/anonFunctionWithoutParens.js.trap index 999173d6a30..154751cda15 100644 --- a/javascript/extractor/tests/flow/output/trap/anonFunctionWithoutParens.js.trap +++ b/javascript/extractor/tests/flow/output/trap/anonFunctionWithoutParens.js.trap @@ -72,20 +72,14 @@ locations_default(#20025,#10000,1,40,1,39) hasLocation(#20024,#20025) toplevels(#20001,0) hasLocation(#20001,#20003) -#20026=@"module;{#10000},1,1" -scopes(#20026,3) -scopenodes(#20001,#20026) -scopenesting(#20026,#20000) -is_module(#20001) -is_es2015_module(#20001) -#20027=* -entry_cfg_node(#20027,#20001) -#20028=@"loc,{#10000},1,1,1,0" -locations_default(#20028,#10000,1,1,1,0) -hasLocation(#20027,#20028) -#20029=* -exit_cfg_node(#20029,#20001) -hasLocation(#20029,#20025) -successor(#20027,#20029) +#20026=* +entry_cfg_node(#20026,#20001) +#20027=@"loc,{#10000},1,1,1,0" +locations_default(#20027,#10000,1,1,1,0) +hasLocation(#20026,#20027) +#20028=* +exit_cfg_node(#20028,#20001) +hasLocation(#20028,#20025) +successor(#20026,#20028) numlines(#10000,1,1,0) filetype(#10000,"javascript") diff --git a/javascript/extractor/tests/flow/output/trap/anonIndexer.js.trap b/javascript/extractor/tests/flow/output/trap/anonIndexer.js.trap index fbfaf50aaee..6f75d1228b7 100644 --- a/javascript/extractor/tests/flow/output/trap/anonIndexer.js.trap +++ b/javascript/extractor/tests/flow/output/trap/anonIndexer.js.trap @@ -85,20 +85,14 @@ toplevels(#20001,0) #20030=@"loc,{#10000},1,1,2,0" locations_default(#20030,#10000,1,1,2,0) hasLocation(#20001,#20030) -#20031=@"module;{#10000},1,1" -scopes(#20031,3) -scopenodes(#20001,#20031) -scopenesting(#20031,#20000) -is_module(#20001) -is_es2015_module(#20001) -#20032=* -entry_cfg_node(#20032,#20001) -#20033=@"loc,{#10000},1,1,1,0" -locations_default(#20033,#10000,1,1,1,0) -hasLocation(#20032,#20033) -#20034=* -exit_cfg_node(#20034,#20001) -hasLocation(#20034,#20029) -successor(#20032,#20034) +#20031=* +entry_cfg_node(#20031,#20001) +#20032=@"loc,{#10000},1,1,1,0" +locations_default(#20032,#10000,1,1,1,0) +hasLocation(#20031,#20032) +#20033=* +exit_cfg_node(#20033,#20001) +hasLocation(#20033,#20029) +successor(#20031,#20033) numlines(#10000,1,1,0) filetype(#10000,"javascript") diff --git a/javascript/extractor/tests/flow/output/trap/declared-module-imports.js.trap b/javascript/extractor/tests/flow/output/trap/declared-module-imports.js.trap index 1eedda348e3..1588e279e76 100644 --- a/javascript/extractor/tests/flow/output/trap/declared-module-imports.js.trap +++ b/javascript/extractor/tests/flow/output/trap/declared-module-imports.js.trap @@ -128,20 +128,14 @@ toplevels(#20001,0) #20045=@"loc,{#10000},1,1,5,0" locations_default(#20045,#10000,1,1,5,0) hasLocation(#20001,#20045) -#20046=@"module;{#10000},1,1" -scopes(#20046,3) -scopenodes(#20001,#20046) -scopenesting(#20046,#20000) -is_module(#20001) -is_es2015_module(#20001) -#20047=* -entry_cfg_node(#20047,#20001) -#20048=@"loc,{#10000},1,1,1,0" -locations_default(#20048,#10000,1,1,1,0) -hasLocation(#20047,#20048) -#20049=* -exit_cfg_node(#20049,#20001) -hasLocation(#20049,#20044) -successor(#20047,#20049) +#20046=* +entry_cfg_node(#20046,#20001) +#20047=@"loc,{#10000},1,1,1,0" +locations_default(#20047,#10000,1,1,1,0) +hasLocation(#20046,#20047) +#20048=* +exit_cfg_node(#20048,#20001) +hasLocation(#20048,#20044) +successor(#20046,#20048) numlines(#10000,4,4,0) filetype(#10000,"javascript") diff --git a/javascript/extractor/tests/flow/output/trap/exportOpaqueType.js.trap b/javascript/extractor/tests/flow/output/trap/exportOpaqueType.js.trap index 5cdd7537e30..5b8c2c2b7d9 100644 --- a/javascript/extractor/tests/flow/output/trap/exportOpaqueType.js.trap +++ b/javascript/extractor/tests/flow/output/trap/exportOpaqueType.js.trap @@ -70,20 +70,14 @@ toplevels(#20001,0) #20024=@"loc,{#10000},1,1,2,0" locations_default(#20024,#10000,1,1,2,0) hasLocation(#20001,#20024) -#20025=@"module;{#10000},1,1" -scopes(#20025,3) -scopenodes(#20001,#20025) -scopenesting(#20025,#20000) -is_module(#20001) -is_es2015_module(#20001) -#20026=* -entry_cfg_node(#20026,#20001) -#20027=@"loc,{#10000},1,1,1,0" -locations_default(#20027,#10000,1,1,1,0) -hasLocation(#20026,#20027) -#20028=* -exit_cfg_node(#20028,#20001) -hasLocation(#20028,#20023) -successor(#20026,#20028) +#20025=* +entry_cfg_node(#20025,#20001) +#20026=@"loc,{#10000},1,1,1,0" +locations_default(#20026,#10000,1,1,1,0) +hasLocation(#20025,#20026) +#20027=* +exit_cfg_node(#20027,#20001) +hasLocation(#20027,#20023) +successor(#20025,#20027) numlines(#10000,1,1,0) filetype(#10000,"javascript") diff --git a/javascript/extractor/tests/flow/output/trap/importTypeInDeclaredModule.js.trap b/javascript/extractor/tests/flow/output/trap/importTypeInDeclaredModule.js.trap index 188033df9a1..8c4b5a30f4a 100644 --- a/javascript/extractor/tests/flow/output/trap/importTypeInDeclaredModule.js.trap +++ b/javascript/extractor/tests/flow/output/trap/importTypeInDeclaredModule.js.trap @@ -205,20 +205,14 @@ toplevels(#20001,0) #20071=@"loc,{#10000},1,1,11,2" locations_default(#20071,#10000,1,1,11,2) hasLocation(#20001,#20071) -#20072=@"module;{#10000},1,1" -scopes(#20072,3) -scopenodes(#20001,#20072) -scopenesting(#20072,#20000) -is_module(#20001) -is_es2015_module(#20001) -#20073=* -entry_cfg_node(#20073,#20001) -#20074=@"loc,{#10000},1,1,1,0" -locations_default(#20074,#10000,1,1,1,0) -hasLocation(#20073,#20074) -#20075=* -exit_cfg_node(#20075,#20001) -hasLocation(#20075,#20070) -successor(#20073,#20075) +#20072=* +entry_cfg_node(#20072,#20001) +#20073=@"loc,{#10000},1,1,1,0" +locations_default(#20073,#10000,1,1,1,0) +hasLocation(#20072,#20073) +#20074=* +exit_cfg_node(#20074,#20001) +hasLocation(#20074,#20070) +successor(#20072,#20074) numlines(#10000,11,10,0) filetype(#10000,"javascript") diff --git a/javascript/extractor/tests/flow/output/trap/variance.js.trap b/javascript/extractor/tests/flow/output/trap/variance.js.trap index b6955639fe9..cd1ea42f228 100644 --- a/javascript/extractor/tests/flow/output/trap/variance.js.trap +++ b/javascript/extractor/tests/flow/output/trap/variance.js.trap @@ -160,95 +160,89 @@ toplevels(#20001,0) #20057=@"loc,{#10000},1,1,7,1" locations_default(#20057,#10000,1,1,7,1) hasLocation(#20001,#20057) -#20058=@"module;{#10000},1,1" -scopes(#20058,3) -scopenodes(#20001,#20058) -scopenesting(#20058,#20000) -is_module(#20001) -is_es2015_module(#20001) -#20059=@"var;{Foo};{#20058}" -variables(#20059,"Foo",#20058) -#20060=@"local_type_name;{Foo};{#20058}" -local_type_names(#20060,"Foo",#20058) -#20061=* -stmts(#20061,26,#20001,0,"class F ... : int\n}") -#20062=@"loc,{#10000},5,1,7,1" -locations_default(#20062,#10000,5,1,7,1) -hasLocation(#20061,#20062) -stmt_containers(#20061,#20001) +#20058=@"var;{Foo};{#20000}" +variables(#20058,"Foo",#20000) +#20059=@"local_type_name;{Foo};{#20000}" +local_type_names(#20059,"Foo",#20000) +#20060=* +stmts(#20060,26,#20001,0,"class F ... : int\n}") +#20061=@"loc,{#10000},5,1,7,1" +locations_default(#20061,#10000,5,1,7,1) +hasLocation(#20060,#20061) +stmt_containers(#20060,#20001) +#20062=* +exprs(#20062,78,#20060,0,"Foo") +hasLocation(#20062,#20043) +enclosing_stmt(#20062,#20060) +expr_containers(#20062,#20001) +literals("Foo","Foo",#20062) +decl(#20062,#20058) +typedecl(#20062,#20059) #20063=* -exprs(#20063,78,#20061,0,"Foo") -hasLocation(#20063,#20043) -enclosing_stmt(#20063,#20061) -expr_containers(#20063,#20001) -literals("Foo","Foo",#20063) -decl(#20063,#20059) -typedecl(#20063,#20060) +scopes(#20063,10) +scopenodes(#20060,#20063) +scopenesting(#20063,#20000) #20064=* -scopes(#20064,10) -scopenodes(#20061,#20064) -scopenesting(#20064,#20058) -#20065=* -properties(#20065,#20061,2,8,"+x: int") -#20066=@"loc,{#10000},6,3,6,9" -locations_default(#20066,#10000,6,3,6,9) -hasLocation(#20065,#20066) +properties(#20064,#20060,2,8,"+x: int") +#20065=@"loc,{#10000},6,3,6,9" +locations_default(#20065,#10000,6,3,6,9) +hasLocation(#20064,#20065) +#20066=* #20067=* +exprs(#20067,0,#20064,0,"x") +hasLocation(#20067,#20049) +expr_containers(#20067,#20066) +literals("x","x",#20067) #20068=* -exprs(#20068,0,#20065,0,"x") -hasLocation(#20068,#20049) -expr_containers(#20068,#20067) -literals("x","x",#20068) -#20069=* -properties(#20069,#20061,3,0,"constructor() {}") -#20070=@"loc,{#10000},5,11,5,10" -locations_default(#20070,#10000,5,11,5,10) -hasLocation(#20069,#20070) +properties(#20068,#20060,3,0,"constructor() {}") +#20069=@"loc,{#10000},5,11,5,10" +locations_default(#20069,#10000,5,11,5,10) +hasLocation(#20068,#20069) +#20070=* +exprs(#20070,0,#20068,0,"constructor") +hasLocation(#20070,#20069) +enclosing_stmt(#20070,#20060) +expr_containers(#20070,#20001) +literals("constructor","constructor",#20070) +exprs(#20066,9,#20068,1,"() {}") +hasLocation(#20066,#20069) +enclosing_stmt(#20066,#20060) +expr_containers(#20066,#20001) #20071=* -exprs(#20071,0,#20069,0,"constructor") -hasLocation(#20071,#20070) -enclosing_stmt(#20071,#20061) -expr_containers(#20071,#20001) -literals("constructor","constructor",#20071) -exprs(#20067,9,#20069,1,"() {}") -hasLocation(#20067,#20070) -enclosing_stmt(#20067,#20061) -expr_containers(#20067,#20001) -#20072=* -scopes(#20072,1) -scopenodes(#20067,#20072) -scopenesting(#20072,#20064) -#20073=@"var;{arguments};{#20072}" -variables(#20073,"arguments",#20072) -is_arguments_object(#20073) +scopes(#20071,1) +scopenodes(#20066,#20071) +scopenesting(#20071,#20063) +#20072=@"var;{arguments};{#20071}" +variables(#20072,"arguments",#20071) +is_arguments_object(#20072) +#20073=* +stmts(#20073,1,#20066,-2,"{}") +hasLocation(#20073,#20069) +stmt_containers(#20073,#20066) +is_method(#20068) #20074=* -stmts(#20074,1,#20067,-2,"{}") -hasLocation(#20074,#20070) -stmt_containers(#20074,#20067) -is_method(#20069) -#20075=* -entry_cfg_node(#20075,#20001) -#20076=@"loc,{#10000},1,1,1,0" -locations_default(#20076,#10000,1,1,1,0) -hasLocation(#20075,#20076) +entry_cfg_node(#20074,#20001) +#20075=@"loc,{#10000},1,1,1,0" +locations_default(#20075,#10000,1,1,1,0) +hasLocation(#20074,#20075) +#20076=* +exit_cfg_node(#20076,#20001) +hasLocation(#20076,#20056) +successor(#20067,#20064) +successor(#20066,#20068) #20077=* -exit_cfg_node(#20077,#20001) -hasLocation(#20077,#20056) -successor(#20068,#20065) -successor(#20067,#20069) +entry_cfg_node(#20077,#20066) +hasLocation(#20077,#20069) +successor(#20064,#20073) #20078=* -entry_cfg_node(#20078,#20067) -hasLocation(#20078,#20070) -successor(#20065,#20074) -#20079=* -exit_cfg_node(#20079,#20067) -hasLocation(#20079,#20070) -successor(#20074,#20079) -successor(#20078,#20068) -successor(#20071,#20067) -successor(#20069,#20061) -successor(#20063,#20071) -successor(#20061,#20077) -successor(#20075,#20063) +exit_cfg_node(#20078,#20066) +hasLocation(#20078,#20069) +successor(#20073,#20078) +successor(#20077,#20067) +successor(#20070,#20066) +successor(#20068,#20060) +successor(#20062,#20070) +successor(#20060,#20076) +successor(#20074,#20062) numlines(#10000,7,6,0) filetype(#10000,"javascript") From df463e51c1a39f36b63ef1a9d72a9a2f3bc70842 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 26 Mar 2024 17:02:47 +0100 Subject: [PATCH 06/39] JS: Extractor: Fix `experimental` flag value for NodeJSDetectorTests --- .../com/semmle/js/extractor/test/NodeJSDetectorTests.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/javascript/extractor/test/com/semmle/js/extractor/test/NodeJSDetectorTests.java b/javascript/extractor/test/com/semmle/js/extractor/test/NodeJSDetectorTests.java index 14d5b323e7c..ead1d126561 100644 --- a/javascript/extractor/test/com/semmle/js/extractor/test/NodeJSDetectorTests.java +++ b/javascript/extractor/test/com/semmle/js/extractor/test/NodeJSDetectorTests.java @@ -11,7 +11,9 @@ import org.junit.Assert; import org.junit.Test; public class NodeJSDetectorTests { - private static final ExtractorConfig CONFIG = new ExtractorConfig(false); + // using `experimental: true` as we do in real extractor, see `extractSource` method + // in `AutoBuild.java` + private static final ExtractorConfig CONFIG = new ExtractorConfig(true); private void isNodeJS(String src, boolean expected) { Result res = JSParser.parse(CONFIG, SourceType.SCRIPT, src, new ExtractionMetrics()); From f33222c83b6cde203153152cf7bf572da7133cb9 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 2 Apr 2024 11:10:53 +0200 Subject: [PATCH 07/39] JS: Add change-note --- .../change-notes/2024-04-02-more-robust-commonjs-detection.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 javascript/ql/lib/change-notes/2024-04-02-more-robust-commonjs-detection.md diff --git a/javascript/ql/lib/change-notes/2024-04-02-more-robust-commonjs-detection.md b/javascript/ql/lib/change-notes/2024-04-02-more-robust-commonjs-detection.md new file mode 100644 index 00000000000..45c3879c39c --- /dev/null +++ b/javascript/ql/lib/change-notes/2024-04-02-more-robust-commonjs-detection.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* Improved detection of whether a file uses CommonJS module system. From ce711f7d2f9fba540dc2a72d70038e9eb39d30a2 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 23 Apr 2024 09:40:44 +0200 Subject: [PATCH 08/39] Python: Move dataflow tests out of experimental --- .../dataflow/TestUtil/DataflowQueryTest.qll | 0 .../dataflow/TestUtil/FlowTest.qll | 0 .../dataflow/TestUtil/LocalFlowStepTest.qll | 0 .../dataflow/TestUtil/MaximalFlowTest.qll | 0 .../dataflow/TestUtil/NormalDataflowTest.qll | 0 .../dataflow/TestUtil/NormalTaintTrackingTest.qll | 0 .../dataflow/TestUtil/RoutingTest.qll | 0 .../dataflow/TestUtil/UnresolvedCalls.qll | 0 .../dataflow/basic/allFlowsConfig.qll | 0 .../dataflow/basic/callGraph.expected | 0 .../{experimental => library-tests}/dataflow/basic/callGraph.ql | 0 .../dataflow/basic/callGraphSinks.expected | 0 .../dataflow/basic/callGraphSinks.ql | 0 .../dataflow/basic/callGraphSources.expected | 0 .../dataflow/basic/callGraphSources.ql | 0 .../dataflow/basic/global.expected | 0 .../test/{experimental => library-tests}/dataflow/basic/global.ql | 0 .../dataflow/basic/globalStep.expected | 0 .../{experimental => library-tests}/dataflow/basic/globalStep.ql | 0 .../{experimental => library-tests}/dataflow/basic/local.expected | 0 .../test/{experimental => library-tests}/dataflow/basic/local.ql | 0 .../dataflow/basic/localFlowStepTest.expected | 0 .../dataflow/basic/localFlowStepTest.ql | 0 .../dataflow/basic/localStep.expected | 0 .../{experimental => library-tests}/dataflow/basic/localStep.ql | 0 .../dataflow/basic/maximalFlowTest.expected | 0 .../dataflow/basic/maximalFlowTest.ql | 0 .../dataflow/basic/maximalFlows.expected | 0 .../dataflow/basic/maximalFlows.ql | 0 .../dataflow/basic/maximalFlowsConfig.qll | 0 .../{experimental => library-tests}/dataflow/basic/sinks.expected | 0 .../test/{experimental => library-tests}/dataflow/basic/sinks.ql | 0 .../dataflow/basic/sources.expected | 0 .../{experimental => library-tests}/dataflow/basic/sources.ql | 0 .../test/{experimental => library-tests}/dataflow/basic/test.py | 0 .../{experimental => library-tests}/dataflow/callGraphConfig.qll | 0 .../dataflow/callgraph_crosstalk/Arguments.expected | 0 .../dataflow/callgraph_crosstalk/Arguments.ql | 0 .../dataflow/callgraph_crosstalk/options | 0 .../dataflow/callgraph_crosstalk/test.py | 0 .../dataflow/calls/DataFlowCallTest.expected | 0 .../dataflow/calls/DataFlowCallTest.ql | 0 .../dataflow/calls/new_cls_param.py | 0 .../test/{experimental => library-tests}/dataflow/calls/test.py | 0 .../dataflow/consistency/class_scope.py | 0 .../dataflow/consistency/modeling-consistency.expected | 0 .../dataflow/consistency/modeling-consistency.ql | 0 .../dataflow/consistency/module.py | 0 .../{experimental => library-tests}/dataflow/consistency/test.py | 0 .../dataflow/coverage-py2/argumentRoutingTest.expected | 0 .../dataflow/coverage-py2/argumentRoutingTest.qlref | 0 .../dataflow/coverage-py2/classes.py | 0 .../{experimental => library-tests}/dataflow/coverage-py2/options | 0 .../dataflow/coverage-py3/argumentRoutingTest.expected | 0 .../dataflow/coverage-py3/argumentRoutingTest.qlref | 0 .../dataflow/coverage-py3/classes.py | 0 .../{experimental => library-tests}/dataflow/coverage-py3/options | 0 .../dataflow/coverage/NormalDataflowTest.expected | 0 .../dataflow/coverage/NormalDataflowTest.ql | 0 .../dataflow/coverage/argumentPassing.py | 0 .../dataflow/coverage/argumentPassing_bad_flow_test.py | 0 .../dataflow/coverage/argumentRoutingTest.expected | 0 .../dataflow/coverage/argumentRoutingTest.ql | 0 .../{experimental => library-tests}/dataflow/coverage/classes.py | 0 .../dataflow/coverage/datamodel.py | 0 .../dataflow/coverage/localFlow.expected | 0 .../dataflow/coverage/localFlow.ql | 0 .../{experimental => library-tests}/dataflow/coverage/loops.py | 0 .../dataflow/coverage/module_level.py | 0 .../{experimental => library-tests}/dataflow/coverage/test.py | 0 .../dataflow/coverage/test_builtins.py | 0 .../dataflow/def-use-flow/def_use_counts.expected | 0 .../dataflow/def-use-flow/def_use_counts.ql | 0 .../dataflow/def-use-flow/def_use_flow.py | 0 .../dataflow/enclosing-callable/EnclosingCallable.expected | 0 .../dataflow/enclosing-callable/EnclosingCallable.ql | 0 .../dataflow/enclosing-callable/class_example.py | 0 .../dataflow/enclosing-callable/generator.py | 0 .../dataflow/exceptions/NormalDataflowTest.expected | 0 .../dataflow/exceptions/NormalDataflowTest.ql | 0 .../{experimental => library-tests}/dataflow/exceptions/test.py | 0 .../dataflow/exceptions/test_group.py | 0 .../dataflow/fieldflow/NormalDataflowTest.expected | 0 .../dataflow/fieldflow/NormalDataflowTest.ql | 0 .../dataflow/fieldflow/UnresolvedCalls.expected | 0 .../dataflow/fieldflow/UnresolvedCalls.ql | 0 .../{experimental => library-tests}/dataflow/fieldflow/options | 0 .../{experimental => library-tests}/dataflow/fieldflow/test.py | 0 .../dataflow/fieldflow/test_dict.py | 0 .../dataflow/fieldflow/test_global.py | 0 .../dataflow/global-flow/accesses.expected | 0 .../dataflow/global-flow/accesses.ql | 0 .../{experimental => library-tests}/dataflow/global-flow/known.py | 0 .../{experimental => library-tests}/dataflow/global-flow/test.py | 0 .../{experimental => library-tests}/dataflow/import-star/deux.py | 0 .../dataflow/import-star/global.expected | 0 .../dataflow/import-star/global.ql | 0 .../{experimental => library-tests}/dataflow/import-star/one.py | 0 .../{experimental => library-tests}/dataflow/import-star/test1.py | 0 .../{experimental => library-tests}/dataflow/import-star/test2.py | 0 .../{experimental => library-tests}/dataflow/import-star/test3.py | 0 .../{experimental => library-tests}/dataflow/import-star/three.py | 0 .../{experimental => library-tests}/dataflow/import-star/trois.py | 0 .../{experimental => library-tests}/dataflow/import-star/two.py | 0 .../{experimental => library-tests}/dataflow/import-star/un.py | 0 .../dataflow/match/NormalDataflowTest.expected | 0 .../dataflow/match/NormalDataflowTest.ql | 0 .../test/{experimental => library-tests}/dataflow/match/test.py | 0 .../dataflow/method-calls/test.expected | 0 .../{experimental => library-tests}/dataflow/method-calls/test.py | 0 .../{experimental => library-tests}/dataflow/method-calls/test.ql | 0 .../dataflow/model-summaries/InlineTaintTest.expected | 0 .../dataflow/model-summaries/InlineTaintTest.ext.yml | 0 .../dataflow/model-summaries/InlineTaintTest.ql | 0 .../dataflow/model-summaries/NormalDataflowTest.expected | 0 .../dataflow/model-summaries/NormalDataflowTest.ext.yml | 0 .../dataflow/model-summaries/NormalDataflowTest.ql | 0 .../dataflow/model-summaries/model_summaries.py | 0 .../dataflow/module-initialization/base.py | 0 .../dataflow/module-initialization/localFlow.expected | 0 .../dataflow/module-initialization/localFlow.ql | 0 .../dataflow/module-initialization/m1.py | 0 .../dataflow/module-initialization/multiphase.py | 0 .../dataflow/module-initialization/test.py | 0 .../dataflow/module-initialization/testOnce.py | 0 python/ql/test/{experimental => library-tests}/dataflow/options | 0 .../dataflow/path-graph/PathNodes.expected | 0 .../dataflow/path-graph/PathNodes.ql | 0 .../{experimental => library-tests}/dataflow/path-graph/test.py | 0 .../{experimental => library-tests}/dataflow/pep_328/__init__.py | 0 .../dataflow/pep_328/package/__init__.py | 0 .../dataflow/pep_328/package/moduleA.py | 0 .../dataflow/pep_328/package/subpackage1/__init__.py | 0 .../dataflow/pep_328/package/subpackage1/moduleX.py | 0 .../dataflow/pep_328/package/subpackage1/moduleY.py | 0 .../dataflow/pep_328/package/subpackage2/__init__.py | 0 .../dataflow/pep_328/package/subpackage2/moduleZ.py | 0 .../{experimental => library-tests}/dataflow/pep_328/start.py | 0 .../dataflow/qll-private-imports/README.md | 0 .../dataflow/qll-private-imports/Test.expected | 0 .../dataflow/qll-private-imports/Test.ql | 0 .../dataflow/qll-private-imports/test.py | 0 .../dataflow/regression/custom_dataflow.expected | 0 .../dataflow/regression/custom_dataflow.ql | 0 .../dataflow/regression/dataflow.expected | 0 .../dataflow/regression/dataflow.ql | 0 .../{experimental => library-tests}/dataflow/regression/module.py | 0 .../{experimental => library-tests}/dataflow/regression/test.py | 0 .../dataflow/sensitive-data/TestSensitiveDataSources.expected | 0 .../dataflow/sensitive-data/TestSensitiveDataSources.ql | 0 .../dataflow/sensitive-data/test.py | 0 .../dataflow/strange-essaflow/test.py | 0 .../dataflow/strange-essaflow/testFlow.expected | 0 .../dataflow/strange-essaflow/testFlow.ql | 0 .../dataflow/summaries-checks/dummy.py | 0 .../dataflow/summaries-checks/invalid-spec.expected | 0 .../dataflow/summaries-checks/invalid-spec.ql | 0 .../dataflow/summaries-checks/missing-attribute-content.expected | 0 .../dataflow/summaries-checks/missing-attribute-content.ql | 0 .../dataflow/summaries/InlineTaintTest.expected | 0 .../dataflow/summaries/InlineTaintTest.ql | 0 .../dataflow/summaries/NormalTaintTrackingTest.expected | 0 .../dataflow/summaries/NormalTaintTrackingTest.ql | 0 .../dataflow/summaries/TestSummaries.qll | 0 .../dataflow/summaries/conflicting_summaries.py | 0 .../dataflow/summaries/extracted_package/__init__.py | 0 .../dataflow/summaries/extracted_package/functions.py | 0 .../dataflow/summaries/summaries.expected | 0 .../dataflow/summaries/summaries.py | 0 .../dataflow/summaries/summaries.ql | 0 .../dataflow/tainttracking/TestTaintLib.qll | 0 .../dataflow/tainttracking/basic/GlobalTaintTracking.expected | 0 .../dataflow/tainttracking/basic/GlobalTaintTracking.ql | 0 .../dataflow/tainttracking/basic/LocalTaintStep.expected | 0 .../dataflow/tainttracking/basic/LocalTaintStep.ql | 0 .../dataflow/tainttracking/basic/test.py | 0 .../tainttracking/commonSanitizer/InlineTaintTest.expected | 0 .../dataflow/tainttracking/commonSanitizer/InlineTaintTest.ql | 0 .../tainttracking/commonSanitizer/test_string_const_compare.py | 0 .../tainttracking/customSanitizer/InlineTaintTest.expected | 0 .../dataflow/tainttracking/customSanitizer/InlineTaintTest.ql | 0 .../dataflow/tainttracking/customSanitizer/test.py | 0 .../dataflow/tainttracking/customSanitizer/test_logical.py | 0 .../dataflow/tainttracking/customSanitizer/test_reference.py | 0 .../defaultAdditionalTaintStep-py3/InlineTaintTest.expected | 0 .../defaultAdditionalTaintStep-py3/InlineTaintTest.ql | 0 .../dataflow/tainttracking/defaultAdditionalTaintStep-py3/options | 0 .../defaultAdditionalTaintStep-py3/test_collections.py | 0 .../tainttracking/defaultAdditionalTaintStep-py3/test_pathlib.py | 0 .../tainttracking/defaultAdditionalTaintStep-py3/test_string.py | 0 .../defaultAdditionalTaintStep-py3/test_unpacking.py | 0 .../defaultAdditionalTaintStep/InlineTaintTest.expected | 0 .../tainttracking/defaultAdditionalTaintStep/InlineTaintTest.ql | 0 .../tainttracking/defaultAdditionalTaintStep/test_async.py | 0 .../tainttracking/defaultAdditionalTaintStep/test_attr.py | 0 .../tainttracking/defaultAdditionalTaintStep/test_collections.py | 0 .../dataflow/tainttracking/defaultAdditionalTaintStep/test_for.py | 0 .../tainttracking/defaultAdditionalTaintStep/test_string.py | 0 .../tainttracking/defaultAdditionalTaintStep/test_unpacking.py | 0 .../tainttracking/defaultAdditionalTaintStep/test_with.py | 0 .../tainttracking/generator-flow/InlineTaintTest.expected | 0 .../dataflow/tainttracking/generator-flow/InlineTaintTest.ql | 0 .../tainttracking/generator-flow/NormalDataflowTest.expected | 0 .../dataflow/tainttracking/generator-flow/NormalDataflowTest.ql | 0 .../dataflow/tainttracking/generator-flow/test_dataflow.py | 0 .../dataflow/tainttracking/generator-flow/test_taint.py | 0 .../dataflow/tainttracking/taintlib.py | 0 .../tainttracking/unwanted-global-flow/InlineTaintTest.expected | 0 .../tainttracking/unwanted-global-flow/InlineTaintTest.ql | 0 .../dataflow/tainttracking/unwanted-global-flow/test.py | 0 .../test/{experimental => library-tests}/dataflow/testConfig.qll | 0 .../{experimental => library-tests}/dataflow/testTaintConfig.qll | 0 .../ql/test/{experimental => library-tests}/dataflow/testlib.py | 0 .../dataflow/typetracking-summaries/TestSummaries.qll | 0 .../dataflow/typetracking-summaries/summaries.py | 0 .../dataflow/typetracking-summaries/tracked.expected | 0 .../dataflow/typetracking-summaries/tracked.ql | 0 .../dataflow/typetracking/attribute_tests.py | 0 .../dataflow/typetracking/content_test.py | 0 .../dataflow/typetracking/import_as_attr.py | 0 .../dataflow/typetracking/import_as_attr_dotted.py | 0 .../dataflow/typetracking/moduleattr.expected | 0 .../dataflow/typetracking/moduleattr.ql | 0 .../dataflow/typetracking/multiple_callables.py | 0 .../dataflow/typetracking/mymodule.py | 0 .../{experimental => library-tests}/dataflow/typetracking/test.py | 0 .../dataflow/typetracking/tracked.expected | 0 .../dataflow/typetracking/tracked.ql | 0 .../dataflow/typetracking_imports/README.md | 0 .../dataflow/typetracking_imports/highlight_problem.expected | 0 .../dataflow/typetracking_imports/highlight_problem.ql | 0 .../dataflow/typetracking_imports/options | 0 .../dataflow/typetracking_imports/pkg/__init__.py | 0 .../dataflow/typetracking_imports/pkg/alias_only_direct.py | 0 .../dataflow/typetracking_imports/pkg/alias_problem.py | 0 .../dataflow/typetracking_imports/pkg/alias_problem_fixed.py | 0 .../dataflow/typetracking_imports/pkg/alias_star.py | 0 .../dataflow/typetracking_imports/pkg/foo_def.py | 0 .../dataflow/typetracking_imports/pkg/other.py | 0 .../dataflow/typetracking_imports/pkg/problem_absolute_import.py | 0 .../dataflow/typetracking_imports/pkg/use.py | 0 .../dataflow/typetracking_imports/pkg/works_absolute_import.py | 0 .../dataflow/typetracking_imports/tracked.expected | 0 .../dataflow/typetracking_imports/tracked.qlref | 0 .../dataflow/use-use-flow/read_explosion.py | 0 .../dataflow/use-use-flow/use-use-counts.expected | 0 .../dataflow/use-use-flow/use-use-counts.ql | 0 .../ql/test/{experimental => library-tests}/dataflow/validTest.py | 0 .../dataflow/variable-capture/CaptureTest.expected | 0 .../dataflow/variable-capture/CaptureTest.ql | 0 .../dataflow/variable-capture/by_value.py | 0 .../variable-capture/dataflow-capture-consistency.expected | 0 .../dataflow/variable-capture/dataflow-capture-consistency.ql | 0 .../dataflow/variable-capture/dict.py | 0 .../dataflow/variable-capture/global.py | 0 .../dataflow/variable-capture/in.py | 0 .../dataflow/variable-capture/nonlocal.py | 0 .../dataflow/variable-capture/test_collections.py | 0 .../dataflow/variable-capture/test_fields.py | 0 .../dataflow/variable-capture/test_library_calls.py | 0 260 files changed, 0 insertions(+), 0 deletions(-) rename python/ql/test/{experimental => library-tests}/dataflow/TestUtil/DataflowQueryTest.qll (100%) rename python/ql/test/{experimental => library-tests}/dataflow/TestUtil/FlowTest.qll (100%) rename python/ql/test/{experimental => library-tests}/dataflow/TestUtil/LocalFlowStepTest.qll (100%) rename python/ql/test/{experimental => library-tests}/dataflow/TestUtil/MaximalFlowTest.qll (100%) rename python/ql/test/{experimental => library-tests}/dataflow/TestUtil/NormalDataflowTest.qll (100%) rename python/ql/test/{experimental => library-tests}/dataflow/TestUtil/NormalTaintTrackingTest.qll (100%) rename python/ql/test/{experimental => library-tests}/dataflow/TestUtil/RoutingTest.qll (100%) rename python/ql/test/{experimental => library-tests}/dataflow/TestUtil/UnresolvedCalls.qll (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/allFlowsConfig.qll (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/callGraph.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/callGraph.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/callGraphSinks.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/callGraphSinks.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/callGraphSources.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/callGraphSources.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/global.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/global.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/globalStep.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/globalStep.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/local.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/local.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/localFlowStepTest.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/localFlowStepTest.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/localStep.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/localStep.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/maximalFlowTest.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/maximalFlowTest.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/maximalFlows.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/maximalFlows.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/maximalFlowsConfig.qll (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/sinks.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/sinks.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/sources.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/sources.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/basic/test.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/callGraphConfig.qll (100%) rename python/ql/test/{experimental => library-tests}/dataflow/callgraph_crosstalk/Arguments.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/callgraph_crosstalk/Arguments.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/callgraph_crosstalk/options (100%) rename python/ql/test/{experimental => library-tests}/dataflow/callgraph_crosstalk/test.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/calls/DataFlowCallTest.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/calls/DataFlowCallTest.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/calls/new_cls_param.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/calls/test.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/consistency/class_scope.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/consistency/modeling-consistency.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/consistency/modeling-consistency.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/consistency/module.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/consistency/test.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/coverage-py2/argumentRoutingTest.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/coverage-py2/argumentRoutingTest.qlref (100%) rename python/ql/test/{experimental => library-tests}/dataflow/coverage-py2/classes.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/coverage-py2/options (100%) rename python/ql/test/{experimental => library-tests}/dataflow/coverage-py3/argumentRoutingTest.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/coverage-py3/argumentRoutingTest.qlref (100%) rename python/ql/test/{experimental => library-tests}/dataflow/coverage-py3/classes.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/coverage-py3/options (100%) rename python/ql/test/{experimental => library-tests}/dataflow/coverage/NormalDataflowTest.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/coverage/NormalDataflowTest.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/coverage/argumentPassing.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/coverage/argumentPassing_bad_flow_test.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/coverage/argumentRoutingTest.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/coverage/argumentRoutingTest.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/coverage/classes.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/coverage/datamodel.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/coverage/localFlow.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/coverage/localFlow.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/coverage/loops.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/coverage/module_level.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/coverage/test.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/coverage/test_builtins.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/def-use-flow/def_use_counts.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/def-use-flow/def_use_counts.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/def-use-flow/def_use_flow.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/enclosing-callable/EnclosingCallable.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/enclosing-callable/EnclosingCallable.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/enclosing-callable/class_example.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/enclosing-callable/generator.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/exceptions/NormalDataflowTest.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/exceptions/NormalDataflowTest.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/exceptions/test.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/exceptions/test_group.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/fieldflow/NormalDataflowTest.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/fieldflow/NormalDataflowTest.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/fieldflow/UnresolvedCalls.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/fieldflow/UnresolvedCalls.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/fieldflow/options (100%) rename python/ql/test/{experimental => library-tests}/dataflow/fieldflow/test.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/fieldflow/test_dict.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/fieldflow/test_global.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/global-flow/accesses.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/global-flow/accesses.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/global-flow/known.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/global-flow/test.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/import-star/deux.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/import-star/global.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/import-star/global.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/import-star/one.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/import-star/test1.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/import-star/test2.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/import-star/test3.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/import-star/three.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/import-star/trois.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/import-star/two.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/import-star/un.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/match/NormalDataflowTest.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/match/NormalDataflowTest.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/match/test.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/method-calls/test.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/method-calls/test.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/method-calls/test.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/model-summaries/InlineTaintTest.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/model-summaries/InlineTaintTest.ext.yml (100%) rename python/ql/test/{experimental => library-tests}/dataflow/model-summaries/InlineTaintTest.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/model-summaries/NormalDataflowTest.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/model-summaries/NormalDataflowTest.ext.yml (100%) rename python/ql/test/{experimental => library-tests}/dataflow/model-summaries/NormalDataflowTest.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/model-summaries/model_summaries.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/module-initialization/base.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/module-initialization/localFlow.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/module-initialization/localFlow.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/module-initialization/m1.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/module-initialization/multiphase.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/module-initialization/test.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/module-initialization/testOnce.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/options (100%) rename python/ql/test/{experimental => library-tests}/dataflow/path-graph/PathNodes.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/path-graph/PathNodes.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/path-graph/test.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/pep_328/__init__.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/pep_328/package/__init__.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/pep_328/package/moduleA.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/pep_328/package/subpackage1/__init__.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/pep_328/package/subpackage1/moduleX.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/pep_328/package/subpackage1/moduleY.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/pep_328/package/subpackage2/__init__.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/pep_328/package/subpackage2/moduleZ.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/pep_328/start.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/qll-private-imports/README.md (100%) rename python/ql/test/{experimental => library-tests}/dataflow/qll-private-imports/Test.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/qll-private-imports/Test.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/qll-private-imports/test.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/regression/custom_dataflow.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/regression/custom_dataflow.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/regression/dataflow.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/regression/dataflow.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/regression/module.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/regression/test.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/sensitive-data/TestSensitiveDataSources.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/sensitive-data/TestSensitiveDataSources.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/sensitive-data/test.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/strange-essaflow/test.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/strange-essaflow/testFlow.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/strange-essaflow/testFlow.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/summaries-checks/dummy.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/summaries-checks/invalid-spec.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/summaries-checks/invalid-spec.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/summaries-checks/missing-attribute-content.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/summaries-checks/missing-attribute-content.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/summaries/InlineTaintTest.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/summaries/InlineTaintTest.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/summaries/NormalTaintTrackingTest.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/summaries/NormalTaintTrackingTest.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/summaries/TestSummaries.qll (100%) rename python/ql/test/{experimental => library-tests}/dataflow/summaries/conflicting_summaries.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/summaries/extracted_package/__init__.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/summaries/extracted_package/functions.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/summaries/summaries.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/summaries/summaries.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/summaries/summaries.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/TestTaintLib.qll (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/basic/GlobalTaintTracking.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/basic/GlobalTaintTracking.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/basic/LocalTaintStep.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/basic/LocalTaintStep.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/basic/test.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/commonSanitizer/InlineTaintTest.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/commonSanitizer/InlineTaintTest.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/commonSanitizer/test_string_const_compare.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/customSanitizer/InlineTaintTest.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/customSanitizer/InlineTaintTest.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/customSanitizer/test.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/customSanitizer/test_logical.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/customSanitizer/test_reference.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/defaultAdditionalTaintStep-py3/InlineTaintTest.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/defaultAdditionalTaintStep-py3/InlineTaintTest.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/defaultAdditionalTaintStep-py3/options (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_collections.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_pathlib.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_string.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_unpacking.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/defaultAdditionalTaintStep/InlineTaintTest.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/defaultAdditionalTaintStep/InlineTaintTest.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/defaultAdditionalTaintStep/test_async.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/defaultAdditionalTaintStep/test_attr.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/defaultAdditionalTaintStep/test_collections.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/defaultAdditionalTaintStep/test_for.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/defaultAdditionalTaintStep/test_string.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/defaultAdditionalTaintStep/test_unpacking.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/defaultAdditionalTaintStep/test_with.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/generator-flow/InlineTaintTest.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/generator-flow/InlineTaintTest.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/generator-flow/NormalDataflowTest.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/generator-flow/NormalDataflowTest.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/generator-flow/test_dataflow.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/generator-flow/test_taint.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/taintlib.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/unwanted-global-flow/InlineTaintTest.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/unwanted-global-flow/InlineTaintTest.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/tainttracking/unwanted-global-flow/test.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/testConfig.qll (100%) rename python/ql/test/{experimental => library-tests}/dataflow/testTaintConfig.qll (100%) rename python/ql/test/{experimental => library-tests}/dataflow/testlib.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking-summaries/TestSummaries.qll (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking-summaries/summaries.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking-summaries/tracked.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking-summaries/tracked.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking/attribute_tests.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking/content_test.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking/import_as_attr.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking/import_as_attr_dotted.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking/moduleattr.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking/moduleattr.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking/multiple_callables.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking/mymodule.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking/test.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking/tracked.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking/tracked.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking_imports/README.md (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking_imports/highlight_problem.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking_imports/highlight_problem.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking_imports/options (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking_imports/pkg/__init__.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking_imports/pkg/alias_only_direct.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking_imports/pkg/alias_problem.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking_imports/pkg/alias_problem_fixed.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking_imports/pkg/alias_star.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking_imports/pkg/foo_def.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking_imports/pkg/other.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking_imports/pkg/problem_absolute_import.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking_imports/pkg/use.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking_imports/pkg/works_absolute_import.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking_imports/tracked.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/typetracking_imports/tracked.qlref (100%) rename python/ql/test/{experimental => library-tests}/dataflow/use-use-flow/read_explosion.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/use-use-flow/use-use-counts.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/use-use-flow/use-use-counts.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/validTest.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/variable-capture/CaptureTest.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/variable-capture/CaptureTest.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/variable-capture/by_value.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/variable-capture/dataflow-capture-consistency.expected (100%) rename python/ql/test/{experimental => library-tests}/dataflow/variable-capture/dataflow-capture-consistency.ql (100%) rename python/ql/test/{experimental => library-tests}/dataflow/variable-capture/dict.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/variable-capture/global.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/variable-capture/in.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/variable-capture/nonlocal.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/variable-capture/test_collections.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/variable-capture/test_fields.py (100%) rename python/ql/test/{experimental => library-tests}/dataflow/variable-capture/test_library_calls.py (100%) diff --git a/python/ql/test/experimental/dataflow/TestUtil/DataflowQueryTest.qll b/python/ql/test/library-tests/dataflow/TestUtil/DataflowQueryTest.qll similarity index 100% rename from python/ql/test/experimental/dataflow/TestUtil/DataflowQueryTest.qll rename to python/ql/test/library-tests/dataflow/TestUtil/DataflowQueryTest.qll diff --git a/python/ql/test/experimental/dataflow/TestUtil/FlowTest.qll b/python/ql/test/library-tests/dataflow/TestUtil/FlowTest.qll similarity index 100% rename from python/ql/test/experimental/dataflow/TestUtil/FlowTest.qll rename to python/ql/test/library-tests/dataflow/TestUtil/FlowTest.qll diff --git a/python/ql/test/experimental/dataflow/TestUtil/LocalFlowStepTest.qll b/python/ql/test/library-tests/dataflow/TestUtil/LocalFlowStepTest.qll similarity index 100% rename from python/ql/test/experimental/dataflow/TestUtil/LocalFlowStepTest.qll rename to python/ql/test/library-tests/dataflow/TestUtil/LocalFlowStepTest.qll diff --git a/python/ql/test/experimental/dataflow/TestUtil/MaximalFlowTest.qll b/python/ql/test/library-tests/dataflow/TestUtil/MaximalFlowTest.qll similarity index 100% rename from python/ql/test/experimental/dataflow/TestUtil/MaximalFlowTest.qll rename to python/ql/test/library-tests/dataflow/TestUtil/MaximalFlowTest.qll diff --git a/python/ql/test/experimental/dataflow/TestUtil/NormalDataflowTest.qll b/python/ql/test/library-tests/dataflow/TestUtil/NormalDataflowTest.qll similarity index 100% rename from python/ql/test/experimental/dataflow/TestUtil/NormalDataflowTest.qll rename to python/ql/test/library-tests/dataflow/TestUtil/NormalDataflowTest.qll diff --git a/python/ql/test/experimental/dataflow/TestUtil/NormalTaintTrackingTest.qll b/python/ql/test/library-tests/dataflow/TestUtil/NormalTaintTrackingTest.qll similarity index 100% rename from python/ql/test/experimental/dataflow/TestUtil/NormalTaintTrackingTest.qll rename to python/ql/test/library-tests/dataflow/TestUtil/NormalTaintTrackingTest.qll diff --git a/python/ql/test/experimental/dataflow/TestUtil/RoutingTest.qll b/python/ql/test/library-tests/dataflow/TestUtil/RoutingTest.qll similarity index 100% rename from python/ql/test/experimental/dataflow/TestUtil/RoutingTest.qll rename to python/ql/test/library-tests/dataflow/TestUtil/RoutingTest.qll diff --git a/python/ql/test/experimental/dataflow/TestUtil/UnresolvedCalls.qll b/python/ql/test/library-tests/dataflow/TestUtil/UnresolvedCalls.qll similarity index 100% rename from python/ql/test/experimental/dataflow/TestUtil/UnresolvedCalls.qll rename to python/ql/test/library-tests/dataflow/TestUtil/UnresolvedCalls.qll diff --git a/python/ql/test/experimental/dataflow/basic/allFlowsConfig.qll b/python/ql/test/library-tests/dataflow/basic/allFlowsConfig.qll similarity index 100% rename from python/ql/test/experimental/dataflow/basic/allFlowsConfig.qll rename to python/ql/test/library-tests/dataflow/basic/allFlowsConfig.qll diff --git a/python/ql/test/experimental/dataflow/basic/callGraph.expected b/python/ql/test/library-tests/dataflow/basic/callGraph.expected similarity index 100% rename from python/ql/test/experimental/dataflow/basic/callGraph.expected rename to python/ql/test/library-tests/dataflow/basic/callGraph.expected diff --git a/python/ql/test/experimental/dataflow/basic/callGraph.ql b/python/ql/test/library-tests/dataflow/basic/callGraph.ql similarity index 100% rename from python/ql/test/experimental/dataflow/basic/callGraph.ql rename to python/ql/test/library-tests/dataflow/basic/callGraph.ql diff --git a/python/ql/test/experimental/dataflow/basic/callGraphSinks.expected b/python/ql/test/library-tests/dataflow/basic/callGraphSinks.expected similarity index 100% rename from python/ql/test/experimental/dataflow/basic/callGraphSinks.expected rename to python/ql/test/library-tests/dataflow/basic/callGraphSinks.expected diff --git a/python/ql/test/experimental/dataflow/basic/callGraphSinks.ql b/python/ql/test/library-tests/dataflow/basic/callGraphSinks.ql similarity index 100% rename from python/ql/test/experimental/dataflow/basic/callGraphSinks.ql rename to python/ql/test/library-tests/dataflow/basic/callGraphSinks.ql diff --git a/python/ql/test/experimental/dataflow/basic/callGraphSources.expected b/python/ql/test/library-tests/dataflow/basic/callGraphSources.expected similarity index 100% rename from python/ql/test/experimental/dataflow/basic/callGraphSources.expected rename to python/ql/test/library-tests/dataflow/basic/callGraphSources.expected diff --git a/python/ql/test/experimental/dataflow/basic/callGraphSources.ql b/python/ql/test/library-tests/dataflow/basic/callGraphSources.ql similarity index 100% rename from python/ql/test/experimental/dataflow/basic/callGraphSources.ql rename to python/ql/test/library-tests/dataflow/basic/callGraphSources.ql diff --git a/python/ql/test/experimental/dataflow/basic/global.expected b/python/ql/test/library-tests/dataflow/basic/global.expected similarity index 100% rename from python/ql/test/experimental/dataflow/basic/global.expected rename to python/ql/test/library-tests/dataflow/basic/global.expected diff --git a/python/ql/test/experimental/dataflow/basic/global.ql b/python/ql/test/library-tests/dataflow/basic/global.ql similarity index 100% rename from python/ql/test/experimental/dataflow/basic/global.ql rename to python/ql/test/library-tests/dataflow/basic/global.ql diff --git a/python/ql/test/experimental/dataflow/basic/globalStep.expected b/python/ql/test/library-tests/dataflow/basic/globalStep.expected similarity index 100% rename from python/ql/test/experimental/dataflow/basic/globalStep.expected rename to python/ql/test/library-tests/dataflow/basic/globalStep.expected diff --git a/python/ql/test/experimental/dataflow/basic/globalStep.ql b/python/ql/test/library-tests/dataflow/basic/globalStep.ql similarity index 100% rename from python/ql/test/experimental/dataflow/basic/globalStep.ql rename to python/ql/test/library-tests/dataflow/basic/globalStep.ql diff --git a/python/ql/test/experimental/dataflow/basic/local.expected b/python/ql/test/library-tests/dataflow/basic/local.expected similarity index 100% rename from python/ql/test/experimental/dataflow/basic/local.expected rename to python/ql/test/library-tests/dataflow/basic/local.expected diff --git a/python/ql/test/experimental/dataflow/basic/local.ql b/python/ql/test/library-tests/dataflow/basic/local.ql similarity index 100% rename from python/ql/test/experimental/dataflow/basic/local.ql rename to python/ql/test/library-tests/dataflow/basic/local.ql diff --git a/python/ql/test/experimental/dataflow/basic/localFlowStepTest.expected b/python/ql/test/library-tests/dataflow/basic/localFlowStepTest.expected similarity index 100% rename from python/ql/test/experimental/dataflow/basic/localFlowStepTest.expected rename to python/ql/test/library-tests/dataflow/basic/localFlowStepTest.expected diff --git a/python/ql/test/experimental/dataflow/basic/localFlowStepTest.ql b/python/ql/test/library-tests/dataflow/basic/localFlowStepTest.ql similarity index 100% rename from python/ql/test/experimental/dataflow/basic/localFlowStepTest.ql rename to python/ql/test/library-tests/dataflow/basic/localFlowStepTest.ql diff --git a/python/ql/test/experimental/dataflow/basic/localStep.expected b/python/ql/test/library-tests/dataflow/basic/localStep.expected similarity index 100% rename from python/ql/test/experimental/dataflow/basic/localStep.expected rename to python/ql/test/library-tests/dataflow/basic/localStep.expected diff --git a/python/ql/test/experimental/dataflow/basic/localStep.ql b/python/ql/test/library-tests/dataflow/basic/localStep.ql similarity index 100% rename from python/ql/test/experimental/dataflow/basic/localStep.ql rename to python/ql/test/library-tests/dataflow/basic/localStep.ql diff --git a/python/ql/test/experimental/dataflow/basic/maximalFlowTest.expected b/python/ql/test/library-tests/dataflow/basic/maximalFlowTest.expected similarity index 100% rename from python/ql/test/experimental/dataflow/basic/maximalFlowTest.expected rename to python/ql/test/library-tests/dataflow/basic/maximalFlowTest.expected diff --git a/python/ql/test/experimental/dataflow/basic/maximalFlowTest.ql b/python/ql/test/library-tests/dataflow/basic/maximalFlowTest.ql similarity index 100% rename from python/ql/test/experimental/dataflow/basic/maximalFlowTest.ql rename to python/ql/test/library-tests/dataflow/basic/maximalFlowTest.ql diff --git a/python/ql/test/experimental/dataflow/basic/maximalFlows.expected b/python/ql/test/library-tests/dataflow/basic/maximalFlows.expected similarity index 100% rename from python/ql/test/experimental/dataflow/basic/maximalFlows.expected rename to python/ql/test/library-tests/dataflow/basic/maximalFlows.expected diff --git a/python/ql/test/experimental/dataflow/basic/maximalFlows.ql b/python/ql/test/library-tests/dataflow/basic/maximalFlows.ql similarity index 100% rename from python/ql/test/experimental/dataflow/basic/maximalFlows.ql rename to python/ql/test/library-tests/dataflow/basic/maximalFlows.ql diff --git a/python/ql/test/experimental/dataflow/basic/maximalFlowsConfig.qll b/python/ql/test/library-tests/dataflow/basic/maximalFlowsConfig.qll similarity index 100% rename from python/ql/test/experimental/dataflow/basic/maximalFlowsConfig.qll rename to python/ql/test/library-tests/dataflow/basic/maximalFlowsConfig.qll diff --git a/python/ql/test/experimental/dataflow/basic/sinks.expected b/python/ql/test/library-tests/dataflow/basic/sinks.expected similarity index 100% rename from python/ql/test/experimental/dataflow/basic/sinks.expected rename to python/ql/test/library-tests/dataflow/basic/sinks.expected diff --git a/python/ql/test/experimental/dataflow/basic/sinks.ql b/python/ql/test/library-tests/dataflow/basic/sinks.ql similarity index 100% rename from python/ql/test/experimental/dataflow/basic/sinks.ql rename to python/ql/test/library-tests/dataflow/basic/sinks.ql diff --git a/python/ql/test/experimental/dataflow/basic/sources.expected b/python/ql/test/library-tests/dataflow/basic/sources.expected similarity index 100% rename from python/ql/test/experimental/dataflow/basic/sources.expected rename to python/ql/test/library-tests/dataflow/basic/sources.expected diff --git a/python/ql/test/experimental/dataflow/basic/sources.ql b/python/ql/test/library-tests/dataflow/basic/sources.ql similarity index 100% rename from python/ql/test/experimental/dataflow/basic/sources.ql rename to python/ql/test/library-tests/dataflow/basic/sources.ql diff --git a/python/ql/test/experimental/dataflow/basic/test.py b/python/ql/test/library-tests/dataflow/basic/test.py similarity index 100% rename from python/ql/test/experimental/dataflow/basic/test.py rename to python/ql/test/library-tests/dataflow/basic/test.py diff --git a/python/ql/test/experimental/dataflow/callGraphConfig.qll b/python/ql/test/library-tests/dataflow/callGraphConfig.qll similarity index 100% rename from python/ql/test/experimental/dataflow/callGraphConfig.qll rename to python/ql/test/library-tests/dataflow/callGraphConfig.qll diff --git a/python/ql/test/experimental/dataflow/callgraph_crosstalk/Arguments.expected b/python/ql/test/library-tests/dataflow/callgraph_crosstalk/Arguments.expected similarity index 100% rename from python/ql/test/experimental/dataflow/callgraph_crosstalk/Arguments.expected rename to python/ql/test/library-tests/dataflow/callgraph_crosstalk/Arguments.expected diff --git a/python/ql/test/experimental/dataflow/callgraph_crosstalk/Arguments.ql b/python/ql/test/library-tests/dataflow/callgraph_crosstalk/Arguments.ql similarity index 100% rename from python/ql/test/experimental/dataflow/callgraph_crosstalk/Arguments.ql rename to python/ql/test/library-tests/dataflow/callgraph_crosstalk/Arguments.ql diff --git a/python/ql/test/experimental/dataflow/callgraph_crosstalk/options b/python/ql/test/library-tests/dataflow/callgraph_crosstalk/options similarity index 100% rename from python/ql/test/experimental/dataflow/callgraph_crosstalk/options rename to python/ql/test/library-tests/dataflow/callgraph_crosstalk/options diff --git a/python/ql/test/experimental/dataflow/callgraph_crosstalk/test.py b/python/ql/test/library-tests/dataflow/callgraph_crosstalk/test.py similarity index 100% rename from python/ql/test/experimental/dataflow/callgraph_crosstalk/test.py rename to python/ql/test/library-tests/dataflow/callgraph_crosstalk/test.py diff --git a/python/ql/test/experimental/dataflow/calls/DataFlowCallTest.expected b/python/ql/test/library-tests/dataflow/calls/DataFlowCallTest.expected similarity index 100% rename from python/ql/test/experimental/dataflow/calls/DataFlowCallTest.expected rename to python/ql/test/library-tests/dataflow/calls/DataFlowCallTest.expected diff --git a/python/ql/test/experimental/dataflow/calls/DataFlowCallTest.ql b/python/ql/test/library-tests/dataflow/calls/DataFlowCallTest.ql similarity index 100% rename from python/ql/test/experimental/dataflow/calls/DataFlowCallTest.ql rename to python/ql/test/library-tests/dataflow/calls/DataFlowCallTest.ql diff --git a/python/ql/test/experimental/dataflow/calls/new_cls_param.py b/python/ql/test/library-tests/dataflow/calls/new_cls_param.py similarity index 100% rename from python/ql/test/experimental/dataflow/calls/new_cls_param.py rename to python/ql/test/library-tests/dataflow/calls/new_cls_param.py diff --git a/python/ql/test/experimental/dataflow/calls/test.py b/python/ql/test/library-tests/dataflow/calls/test.py similarity index 100% rename from python/ql/test/experimental/dataflow/calls/test.py rename to python/ql/test/library-tests/dataflow/calls/test.py diff --git a/python/ql/test/experimental/dataflow/consistency/class_scope.py b/python/ql/test/library-tests/dataflow/consistency/class_scope.py similarity index 100% rename from python/ql/test/experimental/dataflow/consistency/class_scope.py rename to python/ql/test/library-tests/dataflow/consistency/class_scope.py diff --git a/python/ql/test/experimental/dataflow/consistency/modeling-consistency.expected b/python/ql/test/library-tests/dataflow/consistency/modeling-consistency.expected similarity index 100% rename from python/ql/test/experimental/dataflow/consistency/modeling-consistency.expected rename to python/ql/test/library-tests/dataflow/consistency/modeling-consistency.expected diff --git a/python/ql/test/experimental/dataflow/consistency/modeling-consistency.ql b/python/ql/test/library-tests/dataflow/consistency/modeling-consistency.ql similarity index 100% rename from python/ql/test/experimental/dataflow/consistency/modeling-consistency.ql rename to python/ql/test/library-tests/dataflow/consistency/modeling-consistency.ql diff --git a/python/ql/test/experimental/dataflow/consistency/module.py b/python/ql/test/library-tests/dataflow/consistency/module.py similarity index 100% rename from python/ql/test/experimental/dataflow/consistency/module.py rename to python/ql/test/library-tests/dataflow/consistency/module.py diff --git a/python/ql/test/experimental/dataflow/consistency/test.py b/python/ql/test/library-tests/dataflow/consistency/test.py similarity index 100% rename from python/ql/test/experimental/dataflow/consistency/test.py rename to python/ql/test/library-tests/dataflow/consistency/test.py diff --git a/python/ql/test/experimental/dataflow/coverage-py2/argumentRoutingTest.expected b/python/ql/test/library-tests/dataflow/coverage-py2/argumentRoutingTest.expected similarity index 100% rename from python/ql/test/experimental/dataflow/coverage-py2/argumentRoutingTest.expected rename to python/ql/test/library-tests/dataflow/coverage-py2/argumentRoutingTest.expected diff --git a/python/ql/test/experimental/dataflow/coverage-py2/argumentRoutingTest.qlref b/python/ql/test/library-tests/dataflow/coverage-py2/argumentRoutingTest.qlref similarity index 100% rename from python/ql/test/experimental/dataflow/coverage-py2/argumentRoutingTest.qlref rename to python/ql/test/library-tests/dataflow/coverage-py2/argumentRoutingTest.qlref diff --git a/python/ql/test/experimental/dataflow/coverage-py2/classes.py b/python/ql/test/library-tests/dataflow/coverage-py2/classes.py similarity index 100% rename from python/ql/test/experimental/dataflow/coverage-py2/classes.py rename to python/ql/test/library-tests/dataflow/coverage-py2/classes.py diff --git a/python/ql/test/experimental/dataflow/coverage-py2/options b/python/ql/test/library-tests/dataflow/coverage-py2/options similarity index 100% rename from python/ql/test/experimental/dataflow/coverage-py2/options rename to python/ql/test/library-tests/dataflow/coverage-py2/options diff --git a/python/ql/test/experimental/dataflow/coverage-py3/argumentRoutingTest.expected b/python/ql/test/library-tests/dataflow/coverage-py3/argumentRoutingTest.expected similarity index 100% rename from python/ql/test/experimental/dataflow/coverage-py3/argumentRoutingTest.expected rename to python/ql/test/library-tests/dataflow/coverage-py3/argumentRoutingTest.expected diff --git a/python/ql/test/experimental/dataflow/coverage-py3/argumentRoutingTest.qlref b/python/ql/test/library-tests/dataflow/coverage-py3/argumentRoutingTest.qlref similarity index 100% rename from python/ql/test/experimental/dataflow/coverage-py3/argumentRoutingTest.qlref rename to python/ql/test/library-tests/dataflow/coverage-py3/argumentRoutingTest.qlref diff --git a/python/ql/test/experimental/dataflow/coverage-py3/classes.py b/python/ql/test/library-tests/dataflow/coverage-py3/classes.py similarity index 100% rename from python/ql/test/experimental/dataflow/coverage-py3/classes.py rename to python/ql/test/library-tests/dataflow/coverage-py3/classes.py diff --git a/python/ql/test/experimental/dataflow/coverage-py3/options b/python/ql/test/library-tests/dataflow/coverage-py3/options similarity index 100% rename from python/ql/test/experimental/dataflow/coverage-py3/options rename to python/ql/test/library-tests/dataflow/coverage-py3/options diff --git a/python/ql/test/experimental/dataflow/coverage/NormalDataflowTest.expected b/python/ql/test/library-tests/dataflow/coverage/NormalDataflowTest.expected similarity index 100% rename from python/ql/test/experimental/dataflow/coverage/NormalDataflowTest.expected rename to python/ql/test/library-tests/dataflow/coverage/NormalDataflowTest.expected diff --git a/python/ql/test/experimental/dataflow/coverage/NormalDataflowTest.ql b/python/ql/test/library-tests/dataflow/coverage/NormalDataflowTest.ql similarity index 100% rename from python/ql/test/experimental/dataflow/coverage/NormalDataflowTest.ql rename to python/ql/test/library-tests/dataflow/coverage/NormalDataflowTest.ql diff --git a/python/ql/test/experimental/dataflow/coverage/argumentPassing.py b/python/ql/test/library-tests/dataflow/coverage/argumentPassing.py similarity index 100% rename from python/ql/test/experimental/dataflow/coverage/argumentPassing.py rename to python/ql/test/library-tests/dataflow/coverage/argumentPassing.py diff --git a/python/ql/test/experimental/dataflow/coverage/argumentPassing_bad_flow_test.py b/python/ql/test/library-tests/dataflow/coverage/argumentPassing_bad_flow_test.py similarity index 100% rename from python/ql/test/experimental/dataflow/coverage/argumentPassing_bad_flow_test.py rename to python/ql/test/library-tests/dataflow/coverage/argumentPassing_bad_flow_test.py diff --git a/python/ql/test/experimental/dataflow/coverage/argumentRoutingTest.expected b/python/ql/test/library-tests/dataflow/coverage/argumentRoutingTest.expected similarity index 100% rename from python/ql/test/experimental/dataflow/coverage/argumentRoutingTest.expected rename to python/ql/test/library-tests/dataflow/coverage/argumentRoutingTest.expected diff --git a/python/ql/test/experimental/dataflow/coverage/argumentRoutingTest.ql b/python/ql/test/library-tests/dataflow/coverage/argumentRoutingTest.ql similarity index 100% rename from python/ql/test/experimental/dataflow/coverage/argumentRoutingTest.ql rename to python/ql/test/library-tests/dataflow/coverage/argumentRoutingTest.ql diff --git a/python/ql/test/experimental/dataflow/coverage/classes.py b/python/ql/test/library-tests/dataflow/coverage/classes.py similarity index 100% rename from python/ql/test/experimental/dataflow/coverage/classes.py rename to python/ql/test/library-tests/dataflow/coverage/classes.py diff --git a/python/ql/test/experimental/dataflow/coverage/datamodel.py b/python/ql/test/library-tests/dataflow/coverage/datamodel.py similarity index 100% rename from python/ql/test/experimental/dataflow/coverage/datamodel.py rename to python/ql/test/library-tests/dataflow/coverage/datamodel.py diff --git a/python/ql/test/experimental/dataflow/coverage/localFlow.expected b/python/ql/test/library-tests/dataflow/coverage/localFlow.expected similarity index 100% rename from python/ql/test/experimental/dataflow/coverage/localFlow.expected rename to python/ql/test/library-tests/dataflow/coverage/localFlow.expected diff --git a/python/ql/test/experimental/dataflow/coverage/localFlow.ql b/python/ql/test/library-tests/dataflow/coverage/localFlow.ql similarity index 100% rename from python/ql/test/experimental/dataflow/coverage/localFlow.ql rename to python/ql/test/library-tests/dataflow/coverage/localFlow.ql diff --git a/python/ql/test/experimental/dataflow/coverage/loops.py b/python/ql/test/library-tests/dataflow/coverage/loops.py similarity index 100% rename from python/ql/test/experimental/dataflow/coverage/loops.py rename to python/ql/test/library-tests/dataflow/coverage/loops.py diff --git a/python/ql/test/experimental/dataflow/coverage/module_level.py b/python/ql/test/library-tests/dataflow/coverage/module_level.py similarity index 100% rename from python/ql/test/experimental/dataflow/coverage/module_level.py rename to python/ql/test/library-tests/dataflow/coverage/module_level.py diff --git a/python/ql/test/experimental/dataflow/coverage/test.py b/python/ql/test/library-tests/dataflow/coverage/test.py similarity index 100% rename from python/ql/test/experimental/dataflow/coverage/test.py rename to python/ql/test/library-tests/dataflow/coverage/test.py diff --git a/python/ql/test/experimental/dataflow/coverage/test_builtins.py b/python/ql/test/library-tests/dataflow/coverage/test_builtins.py similarity index 100% rename from python/ql/test/experimental/dataflow/coverage/test_builtins.py rename to python/ql/test/library-tests/dataflow/coverage/test_builtins.py diff --git a/python/ql/test/experimental/dataflow/def-use-flow/def_use_counts.expected b/python/ql/test/library-tests/dataflow/def-use-flow/def_use_counts.expected similarity index 100% rename from python/ql/test/experimental/dataflow/def-use-flow/def_use_counts.expected rename to python/ql/test/library-tests/dataflow/def-use-flow/def_use_counts.expected diff --git a/python/ql/test/experimental/dataflow/def-use-flow/def_use_counts.ql b/python/ql/test/library-tests/dataflow/def-use-flow/def_use_counts.ql similarity index 100% rename from python/ql/test/experimental/dataflow/def-use-flow/def_use_counts.ql rename to python/ql/test/library-tests/dataflow/def-use-flow/def_use_counts.ql diff --git a/python/ql/test/experimental/dataflow/def-use-flow/def_use_flow.py b/python/ql/test/library-tests/dataflow/def-use-flow/def_use_flow.py similarity index 100% rename from python/ql/test/experimental/dataflow/def-use-flow/def_use_flow.py rename to python/ql/test/library-tests/dataflow/def-use-flow/def_use_flow.py diff --git a/python/ql/test/experimental/dataflow/enclosing-callable/EnclosingCallable.expected b/python/ql/test/library-tests/dataflow/enclosing-callable/EnclosingCallable.expected similarity index 100% rename from python/ql/test/experimental/dataflow/enclosing-callable/EnclosingCallable.expected rename to python/ql/test/library-tests/dataflow/enclosing-callable/EnclosingCallable.expected diff --git a/python/ql/test/experimental/dataflow/enclosing-callable/EnclosingCallable.ql b/python/ql/test/library-tests/dataflow/enclosing-callable/EnclosingCallable.ql similarity index 100% rename from python/ql/test/experimental/dataflow/enclosing-callable/EnclosingCallable.ql rename to python/ql/test/library-tests/dataflow/enclosing-callable/EnclosingCallable.ql diff --git a/python/ql/test/experimental/dataflow/enclosing-callable/class_example.py b/python/ql/test/library-tests/dataflow/enclosing-callable/class_example.py similarity index 100% rename from python/ql/test/experimental/dataflow/enclosing-callable/class_example.py rename to python/ql/test/library-tests/dataflow/enclosing-callable/class_example.py diff --git a/python/ql/test/experimental/dataflow/enclosing-callable/generator.py b/python/ql/test/library-tests/dataflow/enclosing-callable/generator.py similarity index 100% rename from python/ql/test/experimental/dataflow/enclosing-callable/generator.py rename to python/ql/test/library-tests/dataflow/enclosing-callable/generator.py diff --git a/python/ql/test/experimental/dataflow/exceptions/NormalDataflowTest.expected b/python/ql/test/library-tests/dataflow/exceptions/NormalDataflowTest.expected similarity index 100% rename from python/ql/test/experimental/dataflow/exceptions/NormalDataflowTest.expected rename to python/ql/test/library-tests/dataflow/exceptions/NormalDataflowTest.expected diff --git a/python/ql/test/experimental/dataflow/exceptions/NormalDataflowTest.ql b/python/ql/test/library-tests/dataflow/exceptions/NormalDataflowTest.ql similarity index 100% rename from python/ql/test/experimental/dataflow/exceptions/NormalDataflowTest.ql rename to python/ql/test/library-tests/dataflow/exceptions/NormalDataflowTest.ql diff --git a/python/ql/test/experimental/dataflow/exceptions/test.py b/python/ql/test/library-tests/dataflow/exceptions/test.py similarity index 100% rename from python/ql/test/experimental/dataflow/exceptions/test.py rename to python/ql/test/library-tests/dataflow/exceptions/test.py diff --git a/python/ql/test/experimental/dataflow/exceptions/test_group.py b/python/ql/test/library-tests/dataflow/exceptions/test_group.py similarity index 100% rename from python/ql/test/experimental/dataflow/exceptions/test_group.py rename to python/ql/test/library-tests/dataflow/exceptions/test_group.py diff --git a/python/ql/test/experimental/dataflow/fieldflow/NormalDataflowTest.expected b/python/ql/test/library-tests/dataflow/fieldflow/NormalDataflowTest.expected similarity index 100% rename from python/ql/test/experimental/dataflow/fieldflow/NormalDataflowTest.expected rename to python/ql/test/library-tests/dataflow/fieldflow/NormalDataflowTest.expected diff --git a/python/ql/test/experimental/dataflow/fieldflow/NormalDataflowTest.ql b/python/ql/test/library-tests/dataflow/fieldflow/NormalDataflowTest.ql similarity index 100% rename from python/ql/test/experimental/dataflow/fieldflow/NormalDataflowTest.ql rename to python/ql/test/library-tests/dataflow/fieldflow/NormalDataflowTest.ql diff --git a/python/ql/test/experimental/dataflow/fieldflow/UnresolvedCalls.expected b/python/ql/test/library-tests/dataflow/fieldflow/UnresolvedCalls.expected similarity index 100% rename from python/ql/test/experimental/dataflow/fieldflow/UnresolvedCalls.expected rename to python/ql/test/library-tests/dataflow/fieldflow/UnresolvedCalls.expected diff --git a/python/ql/test/experimental/dataflow/fieldflow/UnresolvedCalls.ql b/python/ql/test/library-tests/dataflow/fieldflow/UnresolvedCalls.ql similarity index 100% rename from python/ql/test/experimental/dataflow/fieldflow/UnresolvedCalls.ql rename to python/ql/test/library-tests/dataflow/fieldflow/UnresolvedCalls.ql diff --git a/python/ql/test/experimental/dataflow/fieldflow/options b/python/ql/test/library-tests/dataflow/fieldflow/options similarity index 100% rename from python/ql/test/experimental/dataflow/fieldflow/options rename to python/ql/test/library-tests/dataflow/fieldflow/options diff --git a/python/ql/test/experimental/dataflow/fieldflow/test.py b/python/ql/test/library-tests/dataflow/fieldflow/test.py similarity index 100% rename from python/ql/test/experimental/dataflow/fieldflow/test.py rename to python/ql/test/library-tests/dataflow/fieldflow/test.py diff --git a/python/ql/test/experimental/dataflow/fieldflow/test_dict.py b/python/ql/test/library-tests/dataflow/fieldflow/test_dict.py similarity index 100% rename from python/ql/test/experimental/dataflow/fieldflow/test_dict.py rename to python/ql/test/library-tests/dataflow/fieldflow/test_dict.py diff --git a/python/ql/test/experimental/dataflow/fieldflow/test_global.py b/python/ql/test/library-tests/dataflow/fieldflow/test_global.py similarity index 100% rename from python/ql/test/experimental/dataflow/fieldflow/test_global.py rename to python/ql/test/library-tests/dataflow/fieldflow/test_global.py diff --git a/python/ql/test/experimental/dataflow/global-flow/accesses.expected b/python/ql/test/library-tests/dataflow/global-flow/accesses.expected similarity index 100% rename from python/ql/test/experimental/dataflow/global-flow/accesses.expected rename to python/ql/test/library-tests/dataflow/global-flow/accesses.expected diff --git a/python/ql/test/experimental/dataflow/global-flow/accesses.ql b/python/ql/test/library-tests/dataflow/global-flow/accesses.ql similarity index 100% rename from python/ql/test/experimental/dataflow/global-flow/accesses.ql rename to python/ql/test/library-tests/dataflow/global-flow/accesses.ql diff --git a/python/ql/test/experimental/dataflow/global-flow/known.py b/python/ql/test/library-tests/dataflow/global-flow/known.py similarity index 100% rename from python/ql/test/experimental/dataflow/global-flow/known.py rename to python/ql/test/library-tests/dataflow/global-flow/known.py diff --git a/python/ql/test/experimental/dataflow/global-flow/test.py b/python/ql/test/library-tests/dataflow/global-flow/test.py similarity index 100% rename from python/ql/test/experimental/dataflow/global-flow/test.py rename to python/ql/test/library-tests/dataflow/global-flow/test.py diff --git a/python/ql/test/experimental/dataflow/import-star/deux.py b/python/ql/test/library-tests/dataflow/import-star/deux.py similarity index 100% rename from python/ql/test/experimental/dataflow/import-star/deux.py rename to python/ql/test/library-tests/dataflow/import-star/deux.py diff --git a/python/ql/test/experimental/dataflow/import-star/global.expected b/python/ql/test/library-tests/dataflow/import-star/global.expected similarity index 100% rename from python/ql/test/experimental/dataflow/import-star/global.expected rename to python/ql/test/library-tests/dataflow/import-star/global.expected diff --git a/python/ql/test/experimental/dataflow/import-star/global.ql b/python/ql/test/library-tests/dataflow/import-star/global.ql similarity index 100% rename from python/ql/test/experimental/dataflow/import-star/global.ql rename to python/ql/test/library-tests/dataflow/import-star/global.ql diff --git a/python/ql/test/experimental/dataflow/import-star/one.py b/python/ql/test/library-tests/dataflow/import-star/one.py similarity index 100% rename from python/ql/test/experimental/dataflow/import-star/one.py rename to python/ql/test/library-tests/dataflow/import-star/one.py diff --git a/python/ql/test/experimental/dataflow/import-star/test1.py b/python/ql/test/library-tests/dataflow/import-star/test1.py similarity index 100% rename from python/ql/test/experimental/dataflow/import-star/test1.py rename to python/ql/test/library-tests/dataflow/import-star/test1.py diff --git a/python/ql/test/experimental/dataflow/import-star/test2.py b/python/ql/test/library-tests/dataflow/import-star/test2.py similarity index 100% rename from python/ql/test/experimental/dataflow/import-star/test2.py rename to python/ql/test/library-tests/dataflow/import-star/test2.py diff --git a/python/ql/test/experimental/dataflow/import-star/test3.py b/python/ql/test/library-tests/dataflow/import-star/test3.py similarity index 100% rename from python/ql/test/experimental/dataflow/import-star/test3.py rename to python/ql/test/library-tests/dataflow/import-star/test3.py diff --git a/python/ql/test/experimental/dataflow/import-star/three.py b/python/ql/test/library-tests/dataflow/import-star/three.py similarity index 100% rename from python/ql/test/experimental/dataflow/import-star/three.py rename to python/ql/test/library-tests/dataflow/import-star/three.py diff --git a/python/ql/test/experimental/dataflow/import-star/trois.py b/python/ql/test/library-tests/dataflow/import-star/trois.py similarity index 100% rename from python/ql/test/experimental/dataflow/import-star/trois.py rename to python/ql/test/library-tests/dataflow/import-star/trois.py diff --git a/python/ql/test/experimental/dataflow/import-star/two.py b/python/ql/test/library-tests/dataflow/import-star/two.py similarity index 100% rename from python/ql/test/experimental/dataflow/import-star/two.py rename to python/ql/test/library-tests/dataflow/import-star/two.py diff --git a/python/ql/test/experimental/dataflow/import-star/un.py b/python/ql/test/library-tests/dataflow/import-star/un.py similarity index 100% rename from python/ql/test/experimental/dataflow/import-star/un.py rename to python/ql/test/library-tests/dataflow/import-star/un.py diff --git a/python/ql/test/experimental/dataflow/match/NormalDataflowTest.expected b/python/ql/test/library-tests/dataflow/match/NormalDataflowTest.expected similarity index 100% rename from python/ql/test/experimental/dataflow/match/NormalDataflowTest.expected rename to python/ql/test/library-tests/dataflow/match/NormalDataflowTest.expected diff --git a/python/ql/test/experimental/dataflow/match/NormalDataflowTest.ql b/python/ql/test/library-tests/dataflow/match/NormalDataflowTest.ql similarity index 100% rename from python/ql/test/experimental/dataflow/match/NormalDataflowTest.ql rename to python/ql/test/library-tests/dataflow/match/NormalDataflowTest.ql diff --git a/python/ql/test/experimental/dataflow/match/test.py b/python/ql/test/library-tests/dataflow/match/test.py similarity index 100% rename from python/ql/test/experimental/dataflow/match/test.py rename to python/ql/test/library-tests/dataflow/match/test.py diff --git a/python/ql/test/experimental/dataflow/method-calls/test.expected b/python/ql/test/library-tests/dataflow/method-calls/test.expected similarity index 100% rename from python/ql/test/experimental/dataflow/method-calls/test.expected rename to python/ql/test/library-tests/dataflow/method-calls/test.expected diff --git a/python/ql/test/experimental/dataflow/method-calls/test.py b/python/ql/test/library-tests/dataflow/method-calls/test.py similarity index 100% rename from python/ql/test/experimental/dataflow/method-calls/test.py rename to python/ql/test/library-tests/dataflow/method-calls/test.py diff --git a/python/ql/test/experimental/dataflow/method-calls/test.ql b/python/ql/test/library-tests/dataflow/method-calls/test.ql similarity index 100% rename from python/ql/test/experimental/dataflow/method-calls/test.ql rename to python/ql/test/library-tests/dataflow/method-calls/test.ql diff --git a/python/ql/test/experimental/dataflow/model-summaries/InlineTaintTest.expected b/python/ql/test/library-tests/dataflow/model-summaries/InlineTaintTest.expected similarity index 100% rename from python/ql/test/experimental/dataflow/model-summaries/InlineTaintTest.expected rename to python/ql/test/library-tests/dataflow/model-summaries/InlineTaintTest.expected diff --git a/python/ql/test/experimental/dataflow/model-summaries/InlineTaintTest.ext.yml b/python/ql/test/library-tests/dataflow/model-summaries/InlineTaintTest.ext.yml similarity index 100% rename from python/ql/test/experimental/dataflow/model-summaries/InlineTaintTest.ext.yml rename to python/ql/test/library-tests/dataflow/model-summaries/InlineTaintTest.ext.yml diff --git a/python/ql/test/experimental/dataflow/model-summaries/InlineTaintTest.ql b/python/ql/test/library-tests/dataflow/model-summaries/InlineTaintTest.ql similarity index 100% rename from python/ql/test/experimental/dataflow/model-summaries/InlineTaintTest.ql rename to python/ql/test/library-tests/dataflow/model-summaries/InlineTaintTest.ql diff --git a/python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.expected b/python/ql/test/library-tests/dataflow/model-summaries/NormalDataflowTest.expected similarity index 100% rename from python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.expected rename to python/ql/test/library-tests/dataflow/model-summaries/NormalDataflowTest.expected diff --git a/python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.ext.yml b/python/ql/test/library-tests/dataflow/model-summaries/NormalDataflowTest.ext.yml similarity index 100% rename from python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.ext.yml rename to python/ql/test/library-tests/dataflow/model-summaries/NormalDataflowTest.ext.yml diff --git a/python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.ql b/python/ql/test/library-tests/dataflow/model-summaries/NormalDataflowTest.ql similarity index 100% rename from python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.ql rename to python/ql/test/library-tests/dataflow/model-summaries/NormalDataflowTest.ql diff --git a/python/ql/test/experimental/dataflow/model-summaries/model_summaries.py b/python/ql/test/library-tests/dataflow/model-summaries/model_summaries.py similarity index 100% rename from python/ql/test/experimental/dataflow/model-summaries/model_summaries.py rename to python/ql/test/library-tests/dataflow/model-summaries/model_summaries.py diff --git a/python/ql/test/experimental/dataflow/module-initialization/base.py b/python/ql/test/library-tests/dataflow/module-initialization/base.py similarity index 100% rename from python/ql/test/experimental/dataflow/module-initialization/base.py rename to python/ql/test/library-tests/dataflow/module-initialization/base.py diff --git a/python/ql/test/experimental/dataflow/module-initialization/localFlow.expected b/python/ql/test/library-tests/dataflow/module-initialization/localFlow.expected similarity index 100% rename from python/ql/test/experimental/dataflow/module-initialization/localFlow.expected rename to python/ql/test/library-tests/dataflow/module-initialization/localFlow.expected diff --git a/python/ql/test/experimental/dataflow/module-initialization/localFlow.ql b/python/ql/test/library-tests/dataflow/module-initialization/localFlow.ql similarity index 100% rename from python/ql/test/experimental/dataflow/module-initialization/localFlow.ql rename to python/ql/test/library-tests/dataflow/module-initialization/localFlow.ql diff --git a/python/ql/test/experimental/dataflow/module-initialization/m1.py b/python/ql/test/library-tests/dataflow/module-initialization/m1.py similarity index 100% rename from python/ql/test/experimental/dataflow/module-initialization/m1.py rename to python/ql/test/library-tests/dataflow/module-initialization/m1.py diff --git a/python/ql/test/experimental/dataflow/module-initialization/multiphase.py b/python/ql/test/library-tests/dataflow/module-initialization/multiphase.py similarity index 100% rename from python/ql/test/experimental/dataflow/module-initialization/multiphase.py rename to python/ql/test/library-tests/dataflow/module-initialization/multiphase.py diff --git a/python/ql/test/experimental/dataflow/module-initialization/test.py b/python/ql/test/library-tests/dataflow/module-initialization/test.py similarity index 100% rename from python/ql/test/experimental/dataflow/module-initialization/test.py rename to python/ql/test/library-tests/dataflow/module-initialization/test.py diff --git a/python/ql/test/experimental/dataflow/module-initialization/testOnce.py b/python/ql/test/library-tests/dataflow/module-initialization/testOnce.py similarity index 100% rename from python/ql/test/experimental/dataflow/module-initialization/testOnce.py rename to python/ql/test/library-tests/dataflow/module-initialization/testOnce.py diff --git a/python/ql/test/experimental/dataflow/options b/python/ql/test/library-tests/dataflow/options similarity index 100% rename from python/ql/test/experimental/dataflow/options rename to python/ql/test/library-tests/dataflow/options diff --git a/python/ql/test/experimental/dataflow/path-graph/PathNodes.expected b/python/ql/test/library-tests/dataflow/path-graph/PathNodes.expected similarity index 100% rename from python/ql/test/experimental/dataflow/path-graph/PathNodes.expected rename to python/ql/test/library-tests/dataflow/path-graph/PathNodes.expected diff --git a/python/ql/test/experimental/dataflow/path-graph/PathNodes.ql b/python/ql/test/library-tests/dataflow/path-graph/PathNodes.ql similarity index 100% rename from python/ql/test/experimental/dataflow/path-graph/PathNodes.ql rename to python/ql/test/library-tests/dataflow/path-graph/PathNodes.ql diff --git a/python/ql/test/experimental/dataflow/path-graph/test.py b/python/ql/test/library-tests/dataflow/path-graph/test.py similarity index 100% rename from python/ql/test/experimental/dataflow/path-graph/test.py rename to python/ql/test/library-tests/dataflow/path-graph/test.py diff --git a/python/ql/test/experimental/dataflow/pep_328/__init__.py b/python/ql/test/library-tests/dataflow/pep_328/__init__.py similarity index 100% rename from python/ql/test/experimental/dataflow/pep_328/__init__.py rename to python/ql/test/library-tests/dataflow/pep_328/__init__.py diff --git a/python/ql/test/experimental/dataflow/pep_328/package/__init__.py b/python/ql/test/library-tests/dataflow/pep_328/package/__init__.py similarity index 100% rename from python/ql/test/experimental/dataflow/pep_328/package/__init__.py rename to python/ql/test/library-tests/dataflow/pep_328/package/__init__.py diff --git a/python/ql/test/experimental/dataflow/pep_328/package/moduleA.py b/python/ql/test/library-tests/dataflow/pep_328/package/moduleA.py similarity index 100% rename from python/ql/test/experimental/dataflow/pep_328/package/moduleA.py rename to python/ql/test/library-tests/dataflow/pep_328/package/moduleA.py diff --git a/python/ql/test/experimental/dataflow/pep_328/package/subpackage1/__init__.py b/python/ql/test/library-tests/dataflow/pep_328/package/subpackage1/__init__.py similarity index 100% rename from python/ql/test/experimental/dataflow/pep_328/package/subpackage1/__init__.py rename to python/ql/test/library-tests/dataflow/pep_328/package/subpackage1/__init__.py diff --git a/python/ql/test/experimental/dataflow/pep_328/package/subpackage1/moduleX.py b/python/ql/test/library-tests/dataflow/pep_328/package/subpackage1/moduleX.py similarity index 100% rename from python/ql/test/experimental/dataflow/pep_328/package/subpackage1/moduleX.py rename to python/ql/test/library-tests/dataflow/pep_328/package/subpackage1/moduleX.py diff --git a/python/ql/test/experimental/dataflow/pep_328/package/subpackage1/moduleY.py b/python/ql/test/library-tests/dataflow/pep_328/package/subpackage1/moduleY.py similarity index 100% rename from python/ql/test/experimental/dataflow/pep_328/package/subpackage1/moduleY.py rename to python/ql/test/library-tests/dataflow/pep_328/package/subpackage1/moduleY.py diff --git a/python/ql/test/experimental/dataflow/pep_328/package/subpackage2/__init__.py b/python/ql/test/library-tests/dataflow/pep_328/package/subpackage2/__init__.py similarity index 100% rename from python/ql/test/experimental/dataflow/pep_328/package/subpackage2/__init__.py rename to python/ql/test/library-tests/dataflow/pep_328/package/subpackage2/__init__.py diff --git a/python/ql/test/experimental/dataflow/pep_328/package/subpackage2/moduleZ.py b/python/ql/test/library-tests/dataflow/pep_328/package/subpackage2/moduleZ.py similarity index 100% rename from python/ql/test/experimental/dataflow/pep_328/package/subpackage2/moduleZ.py rename to python/ql/test/library-tests/dataflow/pep_328/package/subpackage2/moduleZ.py diff --git a/python/ql/test/experimental/dataflow/pep_328/start.py b/python/ql/test/library-tests/dataflow/pep_328/start.py similarity index 100% rename from python/ql/test/experimental/dataflow/pep_328/start.py rename to python/ql/test/library-tests/dataflow/pep_328/start.py diff --git a/python/ql/test/experimental/dataflow/qll-private-imports/README.md b/python/ql/test/library-tests/dataflow/qll-private-imports/README.md similarity index 100% rename from python/ql/test/experimental/dataflow/qll-private-imports/README.md rename to python/ql/test/library-tests/dataflow/qll-private-imports/README.md diff --git a/python/ql/test/experimental/dataflow/qll-private-imports/Test.expected b/python/ql/test/library-tests/dataflow/qll-private-imports/Test.expected similarity index 100% rename from python/ql/test/experimental/dataflow/qll-private-imports/Test.expected rename to python/ql/test/library-tests/dataflow/qll-private-imports/Test.expected diff --git a/python/ql/test/experimental/dataflow/qll-private-imports/Test.ql b/python/ql/test/library-tests/dataflow/qll-private-imports/Test.ql similarity index 100% rename from python/ql/test/experimental/dataflow/qll-private-imports/Test.ql rename to python/ql/test/library-tests/dataflow/qll-private-imports/Test.ql diff --git a/python/ql/test/experimental/dataflow/qll-private-imports/test.py b/python/ql/test/library-tests/dataflow/qll-private-imports/test.py similarity index 100% rename from python/ql/test/experimental/dataflow/qll-private-imports/test.py rename to python/ql/test/library-tests/dataflow/qll-private-imports/test.py diff --git a/python/ql/test/experimental/dataflow/regression/custom_dataflow.expected b/python/ql/test/library-tests/dataflow/regression/custom_dataflow.expected similarity index 100% rename from python/ql/test/experimental/dataflow/regression/custom_dataflow.expected rename to python/ql/test/library-tests/dataflow/regression/custom_dataflow.expected diff --git a/python/ql/test/experimental/dataflow/regression/custom_dataflow.ql b/python/ql/test/library-tests/dataflow/regression/custom_dataflow.ql similarity index 100% rename from python/ql/test/experimental/dataflow/regression/custom_dataflow.ql rename to python/ql/test/library-tests/dataflow/regression/custom_dataflow.ql diff --git a/python/ql/test/experimental/dataflow/regression/dataflow.expected b/python/ql/test/library-tests/dataflow/regression/dataflow.expected similarity index 100% rename from python/ql/test/experimental/dataflow/regression/dataflow.expected rename to python/ql/test/library-tests/dataflow/regression/dataflow.expected diff --git a/python/ql/test/experimental/dataflow/regression/dataflow.ql b/python/ql/test/library-tests/dataflow/regression/dataflow.ql similarity index 100% rename from python/ql/test/experimental/dataflow/regression/dataflow.ql rename to python/ql/test/library-tests/dataflow/regression/dataflow.ql diff --git a/python/ql/test/experimental/dataflow/regression/module.py b/python/ql/test/library-tests/dataflow/regression/module.py similarity index 100% rename from python/ql/test/experimental/dataflow/regression/module.py rename to python/ql/test/library-tests/dataflow/regression/module.py diff --git a/python/ql/test/experimental/dataflow/regression/test.py b/python/ql/test/library-tests/dataflow/regression/test.py similarity index 100% rename from python/ql/test/experimental/dataflow/regression/test.py rename to python/ql/test/library-tests/dataflow/regression/test.py diff --git a/python/ql/test/experimental/dataflow/sensitive-data/TestSensitiveDataSources.expected b/python/ql/test/library-tests/dataflow/sensitive-data/TestSensitiveDataSources.expected similarity index 100% rename from python/ql/test/experimental/dataflow/sensitive-data/TestSensitiveDataSources.expected rename to python/ql/test/library-tests/dataflow/sensitive-data/TestSensitiveDataSources.expected diff --git a/python/ql/test/experimental/dataflow/sensitive-data/TestSensitiveDataSources.ql b/python/ql/test/library-tests/dataflow/sensitive-data/TestSensitiveDataSources.ql similarity index 100% rename from python/ql/test/experimental/dataflow/sensitive-data/TestSensitiveDataSources.ql rename to python/ql/test/library-tests/dataflow/sensitive-data/TestSensitiveDataSources.ql diff --git a/python/ql/test/experimental/dataflow/sensitive-data/test.py b/python/ql/test/library-tests/dataflow/sensitive-data/test.py similarity index 100% rename from python/ql/test/experimental/dataflow/sensitive-data/test.py rename to python/ql/test/library-tests/dataflow/sensitive-data/test.py diff --git a/python/ql/test/experimental/dataflow/strange-essaflow/test.py b/python/ql/test/library-tests/dataflow/strange-essaflow/test.py similarity index 100% rename from python/ql/test/experimental/dataflow/strange-essaflow/test.py rename to python/ql/test/library-tests/dataflow/strange-essaflow/test.py diff --git a/python/ql/test/experimental/dataflow/strange-essaflow/testFlow.expected b/python/ql/test/library-tests/dataflow/strange-essaflow/testFlow.expected similarity index 100% rename from python/ql/test/experimental/dataflow/strange-essaflow/testFlow.expected rename to python/ql/test/library-tests/dataflow/strange-essaflow/testFlow.expected diff --git a/python/ql/test/experimental/dataflow/strange-essaflow/testFlow.ql b/python/ql/test/library-tests/dataflow/strange-essaflow/testFlow.ql similarity index 100% rename from python/ql/test/experimental/dataflow/strange-essaflow/testFlow.ql rename to python/ql/test/library-tests/dataflow/strange-essaflow/testFlow.ql diff --git a/python/ql/test/experimental/dataflow/summaries-checks/dummy.py b/python/ql/test/library-tests/dataflow/summaries-checks/dummy.py similarity index 100% rename from python/ql/test/experimental/dataflow/summaries-checks/dummy.py rename to python/ql/test/library-tests/dataflow/summaries-checks/dummy.py diff --git a/python/ql/test/experimental/dataflow/summaries-checks/invalid-spec.expected b/python/ql/test/library-tests/dataflow/summaries-checks/invalid-spec.expected similarity index 100% rename from python/ql/test/experimental/dataflow/summaries-checks/invalid-spec.expected rename to python/ql/test/library-tests/dataflow/summaries-checks/invalid-spec.expected diff --git a/python/ql/test/experimental/dataflow/summaries-checks/invalid-spec.ql b/python/ql/test/library-tests/dataflow/summaries-checks/invalid-spec.ql similarity index 100% rename from python/ql/test/experimental/dataflow/summaries-checks/invalid-spec.ql rename to python/ql/test/library-tests/dataflow/summaries-checks/invalid-spec.ql diff --git a/python/ql/test/experimental/dataflow/summaries-checks/missing-attribute-content.expected b/python/ql/test/library-tests/dataflow/summaries-checks/missing-attribute-content.expected similarity index 100% rename from python/ql/test/experimental/dataflow/summaries-checks/missing-attribute-content.expected rename to python/ql/test/library-tests/dataflow/summaries-checks/missing-attribute-content.expected diff --git a/python/ql/test/experimental/dataflow/summaries-checks/missing-attribute-content.ql b/python/ql/test/library-tests/dataflow/summaries-checks/missing-attribute-content.ql similarity index 100% rename from python/ql/test/experimental/dataflow/summaries-checks/missing-attribute-content.ql rename to python/ql/test/library-tests/dataflow/summaries-checks/missing-attribute-content.ql diff --git a/python/ql/test/experimental/dataflow/summaries/InlineTaintTest.expected b/python/ql/test/library-tests/dataflow/summaries/InlineTaintTest.expected similarity index 100% rename from python/ql/test/experimental/dataflow/summaries/InlineTaintTest.expected rename to python/ql/test/library-tests/dataflow/summaries/InlineTaintTest.expected diff --git a/python/ql/test/experimental/dataflow/summaries/InlineTaintTest.ql b/python/ql/test/library-tests/dataflow/summaries/InlineTaintTest.ql similarity index 100% rename from python/ql/test/experimental/dataflow/summaries/InlineTaintTest.ql rename to python/ql/test/library-tests/dataflow/summaries/InlineTaintTest.ql diff --git a/python/ql/test/experimental/dataflow/summaries/NormalTaintTrackingTest.expected b/python/ql/test/library-tests/dataflow/summaries/NormalTaintTrackingTest.expected similarity index 100% rename from python/ql/test/experimental/dataflow/summaries/NormalTaintTrackingTest.expected rename to python/ql/test/library-tests/dataflow/summaries/NormalTaintTrackingTest.expected diff --git a/python/ql/test/experimental/dataflow/summaries/NormalTaintTrackingTest.ql b/python/ql/test/library-tests/dataflow/summaries/NormalTaintTrackingTest.ql similarity index 100% rename from python/ql/test/experimental/dataflow/summaries/NormalTaintTrackingTest.ql rename to python/ql/test/library-tests/dataflow/summaries/NormalTaintTrackingTest.ql diff --git a/python/ql/test/experimental/dataflow/summaries/TestSummaries.qll b/python/ql/test/library-tests/dataflow/summaries/TestSummaries.qll similarity index 100% rename from python/ql/test/experimental/dataflow/summaries/TestSummaries.qll rename to python/ql/test/library-tests/dataflow/summaries/TestSummaries.qll diff --git a/python/ql/test/experimental/dataflow/summaries/conflicting_summaries.py b/python/ql/test/library-tests/dataflow/summaries/conflicting_summaries.py similarity index 100% rename from python/ql/test/experimental/dataflow/summaries/conflicting_summaries.py rename to python/ql/test/library-tests/dataflow/summaries/conflicting_summaries.py diff --git a/python/ql/test/experimental/dataflow/summaries/extracted_package/__init__.py b/python/ql/test/library-tests/dataflow/summaries/extracted_package/__init__.py similarity index 100% rename from python/ql/test/experimental/dataflow/summaries/extracted_package/__init__.py rename to python/ql/test/library-tests/dataflow/summaries/extracted_package/__init__.py diff --git a/python/ql/test/experimental/dataflow/summaries/extracted_package/functions.py b/python/ql/test/library-tests/dataflow/summaries/extracted_package/functions.py similarity index 100% rename from python/ql/test/experimental/dataflow/summaries/extracted_package/functions.py rename to python/ql/test/library-tests/dataflow/summaries/extracted_package/functions.py diff --git a/python/ql/test/experimental/dataflow/summaries/summaries.expected b/python/ql/test/library-tests/dataflow/summaries/summaries.expected similarity index 100% rename from python/ql/test/experimental/dataflow/summaries/summaries.expected rename to python/ql/test/library-tests/dataflow/summaries/summaries.expected diff --git a/python/ql/test/experimental/dataflow/summaries/summaries.py b/python/ql/test/library-tests/dataflow/summaries/summaries.py similarity index 100% rename from python/ql/test/experimental/dataflow/summaries/summaries.py rename to python/ql/test/library-tests/dataflow/summaries/summaries.py diff --git a/python/ql/test/experimental/dataflow/summaries/summaries.ql b/python/ql/test/library-tests/dataflow/summaries/summaries.ql similarity index 100% rename from python/ql/test/experimental/dataflow/summaries/summaries.ql rename to python/ql/test/library-tests/dataflow/summaries/summaries.ql diff --git a/python/ql/test/experimental/dataflow/tainttracking/TestTaintLib.qll b/python/ql/test/library-tests/dataflow/tainttracking/TestTaintLib.qll similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/TestTaintLib.qll rename to python/ql/test/library-tests/dataflow/tainttracking/TestTaintLib.qll diff --git a/python/ql/test/experimental/dataflow/tainttracking/basic/GlobalTaintTracking.expected b/python/ql/test/library-tests/dataflow/tainttracking/basic/GlobalTaintTracking.expected similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/basic/GlobalTaintTracking.expected rename to python/ql/test/library-tests/dataflow/tainttracking/basic/GlobalTaintTracking.expected diff --git a/python/ql/test/experimental/dataflow/tainttracking/basic/GlobalTaintTracking.ql b/python/ql/test/library-tests/dataflow/tainttracking/basic/GlobalTaintTracking.ql similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/basic/GlobalTaintTracking.ql rename to python/ql/test/library-tests/dataflow/tainttracking/basic/GlobalTaintTracking.ql diff --git a/python/ql/test/experimental/dataflow/tainttracking/basic/LocalTaintStep.expected b/python/ql/test/library-tests/dataflow/tainttracking/basic/LocalTaintStep.expected similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/basic/LocalTaintStep.expected rename to python/ql/test/library-tests/dataflow/tainttracking/basic/LocalTaintStep.expected diff --git a/python/ql/test/experimental/dataflow/tainttracking/basic/LocalTaintStep.ql b/python/ql/test/library-tests/dataflow/tainttracking/basic/LocalTaintStep.ql similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/basic/LocalTaintStep.ql rename to python/ql/test/library-tests/dataflow/tainttracking/basic/LocalTaintStep.ql diff --git a/python/ql/test/experimental/dataflow/tainttracking/basic/test.py b/python/ql/test/library-tests/dataflow/tainttracking/basic/test.py similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/basic/test.py rename to python/ql/test/library-tests/dataflow/tainttracking/basic/test.py diff --git a/python/ql/test/experimental/dataflow/tainttracking/commonSanitizer/InlineTaintTest.expected b/python/ql/test/library-tests/dataflow/tainttracking/commonSanitizer/InlineTaintTest.expected similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/commonSanitizer/InlineTaintTest.expected rename to python/ql/test/library-tests/dataflow/tainttracking/commonSanitizer/InlineTaintTest.expected diff --git a/python/ql/test/experimental/dataflow/tainttracking/commonSanitizer/InlineTaintTest.ql b/python/ql/test/library-tests/dataflow/tainttracking/commonSanitizer/InlineTaintTest.ql similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/commonSanitizer/InlineTaintTest.ql rename to python/ql/test/library-tests/dataflow/tainttracking/commonSanitizer/InlineTaintTest.ql diff --git a/python/ql/test/experimental/dataflow/tainttracking/commonSanitizer/test_string_const_compare.py b/python/ql/test/library-tests/dataflow/tainttracking/commonSanitizer/test_string_const_compare.py similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/commonSanitizer/test_string_const_compare.py rename to python/ql/test/library-tests/dataflow/tainttracking/commonSanitizer/test_string_const_compare.py diff --git a/python/ql/test/experimental/dataflow/tainttracking/customSanitizer/InlineTaintTest.expected b/python/ql/test/library-tests/dataflow/tainttracking/customSanitizer/InlineTaintTest.expected similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/customSanitizer/InlineTaintTest.expected rename to python/ql/test/library-tests/dataflow/tainttracking/customSanitizer/InlineTaintTest.expected diff --git a/python/ql/test/experimental/dataflow/tainttracking/customSanitizer/InlineTaintTest.ql b/python/ql/test/library-tests/dataflow/tainttracking/customSanitizer/InlineTaintTest.ql similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/customSanitizer/InlineTaintTest.ql rename to python/ql/test/library-tests/dataflow/tainttracking/customSanitizer/InlineTaintTest.ql diff --git a/python/ql/test/experimental/dataflow/tainttracking/customSanitizer/test.py b/python/ql/test/library-tests/dataflow/tainttracking/customSanitizer/test.py similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/customSanitizer/test.py rename to python/ql/test/library-tests/dataflow/tainttracking/customSanitizer/test.py diff --git a/python/ql/test/experimental/dataflow/tainttracking/customSanitizer/test_logical.py b/python/ql/test/library-tests/dataflow/tainttracking/customSanitizer/test_logical.py similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/customSanitizer/test_logical.py rename to python/ql/test/library-tests/dataflow/tainttracking/customSanitizer/test_logical.py diff --git a/python/ql/test/experimental/dataflow/tainttracking/customSanitizer/test_reference.py b/python/ql/test/library-tests/dataflow/tainttracking/customSanitizer/test_reference.py similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/customSanitizer/test_reference.py rename to python/ql/test/library-tests/dataflow/tainttracking/customSanitizer/test_reference.py diff --git a/python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep-py3/InlineTaintTest.expected b/python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep-py3/InlineTaintTest.expected similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep-py3/InlineTaintTest.expected rename to python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep-py3/InlineTaintTest.expected diff --git a/python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep-py3/InlineTaintTest.ql b/python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep-py3/InlineTaintTest.ql similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep-py3/InlineTaintTest.ql rename to python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep-py3/InlineTaintTest.ql diff --git a/python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep-py3/options b/python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep-py3/options similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep-py3/options rename to python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep-py3/options diff --git a/python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_collections.py b/python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_collections.py similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_collections.py rename to python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_collections.py diff --git a/python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_pathlib.py b/python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_pathlib.py similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_pathlib.py rename to python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_pathlib.py diff --git a/python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_string.py b/python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_string.py similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_string.py rename to python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_string.py diff --git a/python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_unpacking.py b/python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_unpacking.py similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_unpacking.py rename to python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_unpacking.py diff --git a/python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/InlineTaintTest.expected b/python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/InlineTaintTest.expected similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/InlineTaintTest.expected rename to python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/InlineTaintTest.expected diff --git a/python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/InlineTaintTest.ql b/python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/InlineTaintTest.ql similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/InlineTaintTest.ql rename to python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/InlineTaintTest.ql diff --git a/python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_async.py b/python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/test_async.py similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_async.py rename to python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/test_async.py diff --git a/python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_attr.py b/python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/test_attr.py similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_attr.py rename to python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/test_attr.py diff --git a/python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_collections.py b/python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/test_collections.py similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_collections.py rename to python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/test_collections.py diff --git a/python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_for.py b/python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/test_for.py similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_for.py rename to python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/test_for.py diff --git a/python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_string.py b/python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/test_string.py similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_string.py rename to python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/test_string.py diff --git a/python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_unpacking.py b/python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/test_unpacking.py similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_unpacking.py rename to python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/test_unpacking.py diff --git a/python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_with.py b/python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/test_with.py similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_with.py rename to python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/test_with.py diff --git a/python/ql/test/experimental/dataflow/tainttracking/generator-flow/InlineTaintTest.expected b/python/ql/test/library-tests/dataflow/tainttracking/generator-flow/InlineTaintTest.expected similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/generator-flow/InlineTaintTest.expected rename to python/ql/test/library-tests/dataflow/tainttracking/generator-flow/InlineTaintTest.expected diff --git a/python/ql/test/experimental/dataflow/tainttracking/generator-flow/InlineTaintTest.ql b/python/ql/test/library-tests/dataflow/tainttracking/generator-flow/InlineTaintTest.ql similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/generator-flow/InlineTaintTest.ql rename to python/ql/test/library-tests/dataflow/tainttracking/generator-flow/InlineTaintTest.ql diff --git a/python/ql/test/experimental/dataflow/tainttracking/generator-flow/NormalDataflowTest.expected b/python/ql/test/library-tests/dataflow/tainttracking/generator-flow/NormalDataflowTest.expected similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/generator-flow/NormalDataflowTest.expected rename to python/ql/test/library-tests/dataflow/tainttracking/generator-flow/NormalDataflowTest.expected diff --git a/python/ql/test/experimental/dataflow/tainttracking/generator-flow/NormalDataflowTest.ql b/python/ql/test/library-tests/dataflow/tainttracking/generator-flow/NormalDataflowTest.ql similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/generator-flow/NormalDataflowTest.ql rename to python/ql/test/library-tests/dataflow/tainttracking/generator-flow/NormalDataflowTest.ql diff --git a/python/ql/test/experimental/dataflow/tainttracking/generator-flow/test_dataflow.py b/python/ql/test/library-tests/dataflow/tainttracking/generator-flow/test_dataflow.py similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/generator-flow/test_dataflow.py rename to python/ql/test/library-tests/dataflow/tainttracking/generator-flow/test_dataflow.py diff --git a/python/ql/test/experimental/dataflow/tainttracking/generator-flow/test_taint.py b/python/ql/test/library-tests/dataflow/tainttracking/generator-flow/test_taint.py similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/generator-flow/test_taint.py rename to python/ql/test/library-tests/dataflow/tainttracking/generator-flow/test_taint.py diff --git a/python/ql/test/experimental/dataflow/tainttracking/taintlib.py b/python/ql/test/library-tests/dataflow/tainttracking/taintlib.py similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/taintlib.py rename to python/ql/test/library-tests/dataflow/tainttracking/taintlib.py diff --git a/python/ql/test/experimental/dataflow/tainttracking/unwanted-global-flow/InlineTaintTest.expected b/python/ql/test/library-tests/dataflow/tainttracking/unwanted-global-flow/InlineTaintTest.expected similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/unwanted-global-flow/InlineTaintTest.expected rename to python/ql/test/library-tests/dataflow/tainttracking/unwanted-global-flow/InlineTaintTest.expected diff --git a/python/ql/test/experimental/dataflow/tainttracking/unwanted-global-flow/InlineTaintTest.ql b/python/ql/test/library-tests/dataflow/tainttracking/unwanted-global-flow/InlineTaintTest.ql similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/unwanted-global-flow/InlineTaintTest.ql rename to python/ql/test/library-tests/dataflow/tainttracking/unwanted-global-flow/InlineTaintTest.ql diff --git a/python/ql/test/experimental/dataflow/tainttracking/unwanted-global-flow/test.py b/python/ql/test/library-tests/dataflow/tainttracking/unwanted-global-flow/test.py similarity index 100% rename from python/ql/test/experimental/dataflow/tainttracking/unwanted-global-flow/test.py rename to python/ql/test/library-tests/dataflow/tainttracking/unwanted-global-flow/test.py diff --git a/python/ql/test/experimental/dataflow/testConfig.qll b/python/ql/test/library-tests/dataflow/testConfig.qll similarity index 100% rename from python/ql/test/experimental/dataflow/testConfig.qll rename to python/ql/test/library-tests/dataflow/testConfig.qll diff --git a/python/ql/test/experimental/dataflow/testTaintConfig.qll b/python/ql/test/library-tests/dataflow/testTaintConfig.qll similarity index 100% rename from python/ql/test/experimental/dataflow/testTaintConfig.qll rename to python/ql/test/library-tests/dataflow/testTaintConfig.qll diff --git a/python/ql/test/experimental/dataflow/testlib.py b/python/ql/test/library-tests/dataflow/testlib.py similarity index 100% rename from python/ql/test/experimental/dataflow/testlib.py rename to python/ql/test/library-tests/dataflow/testlib.py diff --git a/python/ql/test/experimental/dataflow/typetracking-summaries/TestSummaries.qll b/python/ql/test/library-tests/dataflow/typetracking-summaries/TestSummaries.qll similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking-summaries/TestSummaries.qll rename to python/ql/test/library-tests/dataflow/typetracking-summaries/TestSummaries.qll diff --git a/python/ql/test/experimental/dataflow/typetracking-summaries/summaries.py b/python/ql/test/library-tests/dataflow/typetracking-summaries/summaries.py similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking-summaries/summaries.py rename to python/ql/test/library-tests/dataflow/typetracking-summaries/summaries.py diff --git a/python/ql/test/experimental/dataflow/typetracking-summaries/tracked.expected b/python/ql/test/library-tests/dataflow/typetracking-summaries/tracked.expected similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking-summaries/tracked.expected rename to python/ql/test/library-tests/dataflow/typetracking-summaries/tracked.expected diff --git a/python/ql/test/experimental/dataflow/typetracking-summaries/tracked.ql b/python/ql/test/library-tests/dataflow/typetracking-summaries/tracked.ql similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking-summaries/tracked.ql rename to python/ql/test/library-tests/dataflow/typetracking-summaries/tracked.ql diff --git a/python/ql/test/experimental/dataflow/typetracking/attribute_tests.py b/python/ql/test/library-tests/dataflow/typetracking/attribute_tests.py similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking/attribute_tests.py rename to python/ql/test/library-tests/dataflow/typetracking/attribute_tests.py diff --git a/python/ql/test/experimental/dataflow/typetracking/content_test.py b/python/ql/test/library-tests/dataflow/typetracking/content_test.py similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking/content_test.py rename to python/ql/test/library-tests/dataflow/typetracking/content_test.py diff --git a/python/ql/test/experimental/dataflow/typetracking/import_as_attr.py b/python/ql/test/library-tests/dataflow/typetracking/import_as_attr.py similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking/import_as_attr.py rename to python/ql/test/library-tests/dataflow/typetracking/import_as_attr.py diff --git a/python/ql/test/experimental/dataflow/typetracking/import_as_attr_dotted.py b/python/ql/test/library-tests/dataflow/typetracking/import_as_attr_dotted.py similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking/import_as_attr_dotted.py rename to python/ql/test/library-tests/dataflow/typetracking/import_as_attr_dotted.py diff --git a/python/ql/test/experimental/dataflow/typetracking/moduleattr.expected b/python/ql/test/library-tests/dataflow/typetracking/moduleattr.expected similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking/moduleattr.expected rename to python/ql/test/library-tests/dataflow/typetracking/moduleattr.expected diff --git a/python/ql/test/experimental/dataflow/typetracking/moduleattr.ql b/python/ql/test/library-tests/dataflow/typetracking/moduleattr.ql similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking/moduleattr.ql rename to python/ql/test/library-tests/dataflow/typetracking/moduleattr.ql diff --git a/python/ql/test/experimental/dataflow/typetracking/multiple_callables.py b/python/ql/test/library-tests/dataflow/typetracking/multiple_callables.py similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking/multiple_callables.py rename to python/ql/test/library-tests/dataflow/typetracking/multiple_callables.py diff --git a/python/ql/test/experimental/dataflow/typetracking/mymodule.py b/python/ql/test/library-tests/dataflow/typetracking/mymodule.py similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking/mymodule.py rename to python/ql/test/library-tests/dataflow/typetracking/mymodule.py diff --git a/python/ql/test/experimental/dataflow/typetracking/test.py b/python/ql/test/library-tests/dataflow/typetracking/test.py similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking/test.py rename to python/ql/test/library-tests/dataflow/typetracking/test.py diff --git a/python/ql/test/experimental/dataflow/typetracking/tracked.expected b/python/ql/test/library-tests/dataflow/typetracking/tracked.expected similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking/tracked.expected rename to python/ql/test/library-tests/dataflow/typetracking/tracked.expected diff --git a/python/ql/test/experimental/dataflow/typetracking/tracked.ql b/python/ql/test/library-tests/dataflow/typetracking/tracked.ql similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking/tracked.ql rename to python/ql/test/library-tests/dataflow/typetracking/tracked.ql diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/README.md b/python/ql/test/library-tests/dataflow/typetracking_imports/README.md similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking_imports/README.md rename to python/ql/test/library-tests/dataflow/typetracking_imports/README.md diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/highlight_problem.expected b/python/ql/test/library-tests/dataflow/typetracking_imports/highlight_problem.expected similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking_imports/highlight_problem.expected rename to python/ql/test/library-tests/dataflow/typetracking_imports/highlight_problem.expected diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/highlight_problem.ql b/python/ql/test/library-tests/dataflow/typetracking_imports/highlight_problem.ql similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking_imports/highlight_problem.ql rename to python/ql/test/library-tests/dataflow/typetracking_imports/highlight_problem.ql diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/options b/python/ql/test/library-tests/dataflow/typetracking_imports/options similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking_imports/options rename to python/ql/test/library-tests/dataflow/typetracking_imports/options diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/__init__.py b/python/ql/test/library-tests/dataflow/typetracking_imports/pkg/__init__.py similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking_imports/pkg/__init__.py rename to python/ql/test/library-tests/dataflow/typetracking_imports/pkg/__init__.py diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_only_direct.py b/python/ql/test/library-tests/dataflow/typetracking_imports/pkg/alias_only_direct.py similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_only_direct.py rename to python/ql/test/library-tests/dataflow/typetracking_imports/pkg/alias_only_direct.py diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_problem.py b/python/ql/test/library-tests/dataflow/typetracking_imports/pkg/alias_problem.py similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_problem.py rename to python/ql/test/library-tests/dataflow/typetracking_imports/pkg/alias_problem.py diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_problem_fixed.py b/python/ql/test/library-tests/dataflow/typetracking_imports/pkg/alias_problem_fixed.py similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_problem_fixed.py rename to python/ql/test/library-tests/dataflow/typetracking_imports/pkg/alias_problem_fixed.py diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_star.py b/python/ql/test/library-tests/dataflow/typetracking_imports/pkg/alias_star.py similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking_imports/pkg/alias_star.py rename to python/ql/test/library-tests/dataflow/typetracking_imports/pkg/alias_star.py diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/foo_def.py b/python/ql/test/library-tests/dataflow/typetracking_imports/pkg/foo_def.py similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking_imports/pkg/foo_def.py rename to python/ql/test/library-tests/dataflow/typetracking_imports/pkg/foo_def.py diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/other.py b/python/ql/test/library-tests/dataflow/typetracking_imports/pkg/other.py similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking_imports/pkg/other.py rename to python/ql/test/library-tests/dataflow/typetracking_imports/pkg/other.py diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/problem_absolute_import.py b/python/ql/test/library-tests/dataflow/typetracking_imports/pkg/problem_absolute_import.py similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking_imports/pkg/problem_absolute_import.py rename to python/ql/test/library-tests/dataflow/typetracking_imports/pkg/problem_absolute_import.py diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/use.py b/python/ql/test/library-tests/dataflow/typetracking_imports/pkg/use.py similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking_imports/pkg/use.py rename to python/ql/test/library-tests/dataflow/typetracking_imports/pkg/use.py diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/pkg/works_absolute_import.py b/python/ql/test/library-tests/dataflow/typetracking_imports/pkg/works_absolute_import.py similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking_imports/pkg/works_absolute_import.py rename to python/ql/test/library-tests/dataflow/typetracking_imports/pkg/works_absolute_import.py diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/tracked.expected b/python/ql/test/library-tests/dataflow/typetracking_imports/tracked.expected similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking_imports/tracked.expected rename to python/ql/test/library-tests/dataflow/typetracking_imports/tracked.expected diff --git a/python/ql/test/experimental/dataflow/typetracking_imports/tracked.qlref b/python/ql/test/library-tests/dataflow/typetracking_imports/tracked.qlref similarity index 100% rename from python/ql/test/experimental/dataflow/typetracking_imports/tracked.qlref rename to python/ql/test/library-tests/dataflow/typetracking_imports/tracked.qlref diff --git a/python/ql/test/experimental/dataflow/use-use-flow/read_explosion.py b/python/ql/test/library-tests/dataflow/use-use-flow/read_explosion.py similarity index 100% rename from python/ql/test/experimental/dataflow/use-use-flow/read_explosion.py rename to python/ql/test/library-tests/dataflow/use-use-flow/read_explosion.py diff --git a/python/ql/test/experimental/dataflow/use-use-flow/use-use-counts.expected b/python/ql/test/library-tests/dataflow/use-use-flow/use-use-counts.expected similarity index 100% rename from python/ql/test/experimental/dataflow/use-use-flow/use-use-counts.expected rename to python/ql/test/library-tests/dataflow/use-use-flow/use-use-counts.expected diff --git a/python/ql/test/experimental/dataflow/use-use-flow/use-use-counts.ql b/python/ql/test/library-tests/dataflow/use-use-flow/use-use-counts.ql similarity index 100% rename from python/ql/test/experimental/dataflow/use-use-flow/use-use-counts.ql rename to python/ql/test/library-tests/dataflow/use-use-flow/use-use-counts.ql diff --git a/python/ql/test/experimental/dataflow/validTest.py b/python/ql/test/library-tests/dataflow/validTest.py similarity index 100% rename from python/ql/test/experimental/dataflow/validTest.py rename to python/ql/test/library-tests/dataflow/validTest.py diff --git a/python/ql/test/experimental/dataflow/variable-capture/CaptureTest.expected b/python/ql/test/library-tests/dataflow/variable-capture/CaptureTest.expected similarity index 100% rename from python/ql/test/experimental/dataflow/variable-capture/CaptureTest.expected rename to python/ql/test/library-tests/dataflow/variable-capture/CaptureTest.expected diff --git a/python/ql/test/experimental/dataflow/variable-capture/CaptureTest.ql b/python/ql/test/library-tests/dataflow/variable-capture/CaptureTest.ql similarity index 100% rename from python/ql/test/experimental/dataflow/variable-capture/CaptureTest.ql rename to python/ql/test/library-tests/dataflow/variable-capture/CaptureTest.ql diff --git a/python/ql/test/experimental/dataflow/variable-capture/by_value.py b/python/ql/test/library-tests/dataflow/variable-capture/by_value.py similarity index 100% rename from python/ql/test/experimental/dataflow/variable-capture/by_value.py rename to python/ql/test/library-tests/dataflow/variable-capture/by_value.py diff --git a/python/ql/test/experimental/dataflow/variable-capture/dataflow-capture-consistency.expected b/python/ql/test/library-tests/dataflow/variable-capture/dataflow-capture-consistency.expected similarity index 100% rename from python/ql/test/experimental/dataflow/variable-capture/dataflow-capture-consistency.expected rename to python/ql/test/library-tests/dataflow/variable-capture/dataflow-capture-consistency.expected diff --git a/python/ql/test/experimental/dataflow/variable-capture/dataflow-capture-consistency.ql b/python/ql/test/library-tests/dataflow/variable-capture/dataflow-capture-consistency.ql similarity index 100% rename from python/ql/test/experimental/dataflow/variable-capture/dataflow-capture-consistency.ql rename to python/ql/test/library-tests/dataflow/variable-capture/dataflow-capture-consistency.ql diff --git a/python/ql/test/experimental/dataflow/variable-capture/dict.py b/python/ql/test/library-tests/dataflow/variable-capture/dict.py similarity index 100% rename from python/ql/test/experimental/dataflow/variable-capture/dict.py rename to python/ql/test/library-tests/dataflow/variable-capture/dict.py diff --git a/python/ql/test/experimental/dataflow/variable-capture/global.py b/python/ql/test/library-tests/dataflow/variable-capture/global.py similarity index 100% rename from python/ql/test/experimental/dataflow/variable-capture/global.py rename to python/ql/test/library-tests/dataflow/variable-capture/global.py diff --git a/python/ql/test/experimental/dataflow/variable-capture/in.py b/python/ql/test/library-tests/dataflow/variable-capture/in.py similarity index 100% rename from python/ql/test/experimental/dataflow/variable-capture/in.py rename to python/ql/test/library-tests/dataflow/variable-capture/in.py diff --git a/python/ql/test/experimental/dataflow/variable-capture/nonlocal.py b/python/ql/test/library-tests/dataflow/variable-capture/nonlocal.py similarity index 100% rename from python/ql/test/experimental/dataflow/variable-capture/nonlocal.py rename to python/ql/test/library-tests/dataflow/variable-capture/nonlocal.py diff --git a/python/ql/test/experimental/dataflow/variable-capture/test_collections.py b/python/ql/test/library-tests/dataflow/variable-capture/test_collections.py similarity index 100% rename from python/ql/test/experimental/dataflow/variable-capture/test_collections.py rename to python/ql/test/library-tests/dataflow/variable-capture/test_collections.py diff --git a/python/ql/test/experimental/dataflow/variable-capture/test_fields.py b/python/ql/test/library-tests/dataflow/variable-capture/test_fields.py similarity index 100% rename from python/ql/test/experimental/dataflow/variable-capture/test_fields.py rename to python/ql/test/library-tests/dataflow/variable-capture/test_fields.py diff --git a/python/ql/test/experimental/dataflow/variable-capture/test_library_calls.py b/python/ql/test/library-tests/dataflow/variable-capture/test_library_calls.py similarity index 100% rename from python/ql/test/experimental/dataflow/variable-capture/test_library_calls.py rename to python/ql/test/library-tests/dataflow/variable-capture/test_library_calls.py From e0e405bb313095649ffd7578062575533656916f Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Fri, 19 Apr 2024 10:39:45 +0200 Subject: [PATCH 09/39] Python: replace dataflow-test location in files --- config/identical-files.json | 6 +++--- .../lib/semmle/python/dataflow/new/SensitiveDataSources.qll | 2 +- .../semmle/python/dataflow/new/internal/DataFlowPublic.qll | 2 +- python/ql/test/library-tests/dataflow/coverage/datamodel.py | 2 +- python/ql/test/library-tests/dataflow/coverage/test.py | 2 +- .../ql/test/library-tests/frameworks/stdlib-py3/Decoding.py | 2 +- .../ql/test/library-tests/frameworks/stdlib-py3/Encoding.py | 2 +- python/ql/test/library-tests/frameworks/stdlib/Decoding.py | 2 +- python/ql/test/library-tests/frameworks/stdlib/Encoding.py | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/config/identical-files.json b/config/identical-files.json index a8b1368f1af..d810e30c0c8 100644 --- a/config/identical-files.json +++ b/config/identical-files.json @@ -362,7 +362,7 @@ "java/ql/lib/semmle/code/java/security/internal/EncryptionKeySizes.qll" ], "Python model summaries test extension": [ - "python/ql/test/experimental/dataflow/model-summaries/InlineTaintTest.ext.yml", - "python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.ext.yml" + "python/ql/test/library-tests/dataflow/model-summaries/InlineTaintTest.ext.yml", + "python/ql/test/library-tests/dataflow/model-summaries/NormalDataflowTest.ext.yml" ] -} \ No newline at end of file +} diff --git a/python/ql/lib/semmle/python/dataflow/new/SensitiveDataSources.qll b/python/ql/lib/semmle/python/dataflow/new/SensitiveDataSources.qll index 705c4476fb1..c12358f6db9 100644 --- a/python/ql/lib/semmle/python/dataflow/new/SensitiveDataSources.qll +++ b/python/ql/lib/semmle/python/dataflow/new/SensitiveDataSources.qll @@ -89,7 +89,7 @@ private module SensitiveDataModeling { */ DataFlow::Node sensitiveLookupStringConst(SensitiveDataClassification classification) { // Note: If this is implemented with type-tracking, we will get cross-talk as - // illustrated in python/ql/test/experimental/dataflow/sensitive-data/test.py + // illustrated in python/ql/test/library-tests/dataflow/sensitive-data/test.py exists(DataFlow::LocalSourceNode source | source.asExpr().(StringLiteral).getText() = sensitiveString(classification) and source.flowsTo(result) diff --git a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll index 9b9caf28ad3..4192241d619 100644 --- a/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll +++ b/python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll @@ -638,7 +638,7 @@ newtype TContent = // name = any(AccessPathToken a).getAnArgument("Attribute") // instead we use a qltest to alert if we write a new summary in QL that uses an // attribute -- see - // python/ql/test/experimental/dataflow/summaries-checks/missing-attribute-content.ql + // python/ql/test/library-tests/dataflow/summaries-checks/missing-attribute-content.ql attr in ["re", "string", "pattern"] or // diff --git a/python/ql/test/library-tests/dataflow/coverage/datamodel.py b/python/ql/test/library-tests/dataflow/coverage/datamodel.py index 89d9b0a819a..22111a0abbd 100644 --- a/python/ql/test/library-tests/dataflow/coverage/datamodel.py +++ b/python/ql/test/library-tests/dataflow/coverage/datamodel.py @@ -6,7 +6,7 @@ # A thorough covering of methods in that document is found in classes.py. # # Intended sources should be the variable `SOURCE` and intended sinks should be -# arguments to the function `SINK` (see python/ql/test/experimental/dataflow/testConfig.qll). +# arguments to the function `SINK` (see python/ql/test/library-tests/dataflow/testConfig.qll). import sys import os diff --git a/python/ql/test/library-tests/dataflow/coverage/test.py b/python/ql/test/library-tests/dataflow/coverage/test.py index eb14fb1dd0b..a2ea9cccd67 100644 --- a/python/ql/test/library-tests/dataflow/coverage/test.py +++ b/python/ql/test/library-tests/dataflow/coverage/test.py @@ -2,7 +2,7 @@ # Headings refer to https://docs.python.org/3/reference/expressions.html, # and are selected whenever they incur dataflow. # Intended sources should be the variable `SOURCE` and intended sinks should be -# arguments to the function `SINK` (see python/ql/test/experimental/dataflow/testConfig.qll). +# arguments to the function `SINK` (see python/ql/test/library-tests/dataflow/testConfig.qll). # # Functions whose name ends with "_with_local_flow" will also be tested for local flow. # diff --git a/python/ql/test/library-tests/frameworks/stdlib-py3/Decoding.py b/python/ql/test/library-tests/frameworks/stdlib-py3/Decoding.py index 11cca46bdf7..ec82bd2f1a7 100644 --- a/python/ql/test/library-tests/frameworks/stdlib-py3/Decoding.py +++ b/python/ql/test/library-tests/frameworks/stdlib-py3/Decoding.py @@ -1,6 +1,6 @@ import base64 -# TODO: These tests should be merged with python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_string.py +# TODO: These tests should be merged with python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_string.py base64.a85decode(payload) # $ decodeInput=payload decodeOutput=base64.a85decode(..) decodeFormat=Ascii85 base64.b85decode(payload) # $ decodeInput=payload decodeOutput=base64.b85decode(..) decodeFormat=Base85 base64.decodebytes(payload) # $ decodeInput=payload decodeOutput=base64.decodebytes(..) decodeFormat=Base64 diff --git a/python/ql/test/library-tests/frameworks/stdlib-py3/Encoding.py b/python/ql/test/library-tests/frameworks/stdlib-py3/Encoding.py index 7aea6a5e579..493e22a6048 100644 --- a/python/ql/test/library-tests/frameworks/stdlib-py3/Encoding.py +++ b/python/ql/test/library-tests/frameworks/stdlib-py3/Encoding.py @@ -1,6 +1,6 @@ import base64 -# TODO: These tests should be merged with python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_string.py +# TODO: These tests should be merged with python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_string.py base64.a85encode(bs) # $ encodeInput=bs encodeOutput=base64.a85encode(..) encodeFormat=Ascii85 base64.b85encode(bs)# $ encodeInput=bs encodeOutput=base64.b85encode(..) encodeFormat=Base85 base64.encodebytes(bs)# $ encodeInput=bs encodeOutput=base64.encodebytes(..) encodeFormat=Base64 diff --git a/python/ql/test/library-tests/frameworks/stdlib/Decoding.py b/python/ql/test/library-tests/frameworks/stdlib/Decoding.py index a977cb2793a..6cd0be5740c 100644 --- a/python/ql/test/library-tests/frameworks/stdlib/Decoding.py +++ b/python/ql/test/library-tests/frameworks/stdlib/Decoding.py @@ -26,7 +26,7 @@ marshal.loads(payload) # $ decodeInput=payload decodeOutput=marshal.loads(..) d shelve.open(filepath) # $ decodeInput=filepath decodeOutput=shelve.open(..) decodeFormat=pickle decodeMayExecuteInput getAPathArgument=filepath shelve.open(filename=filepath) # $ decodeInput=filepath decodeOutput=shelve.open(..) decodeFormat=pickle decodeMayExecuteInput getAPathArgument=filepath -# TODO: These tests should be merged with python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_string.py +# TODO: These tests should be merged with python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/test_string.py base64.b64decode(payload) # $ decodeInput=payload decodeOutput=base64.b64decode(..) decodeFormat=Base64 base64.standard_b64decode(payload) # $ decodeInput=payload decodeOutput=base64.standard_b64decode(..) decodeFormat=Base64 base64.urlsafe_b64decode(payload) # $ decodeInput=payload decodeOutput=base64.urlsafe_b64decode(..) decodeFormat=Base64 diff --git a/python/ql/test/library-tests/frameworks/stdlib/Encoding.py b/python/ql/test/library-tests/frameworks/stdlib/Encoding.py index 38f4372faa2..5573709a05f 100644 --- a/python/ql/test/library-tests/frameworks/stdlib/Encoding.py +++ b/python/ql/test/library-tests/frameworks/stdlib/Encoding.py @@ -5,7 +5,7 @@ import base64 pickle.dumps(obj) # $ MISSING: encodeInput=obj encodeOutput=pickle.dumps(..) encodeFormat=pickle encodeMayExecuteInput marshal.dumps(obj) # $ MISSING: encodeInput=obj encodeOutput=marshal.dumps(..) encodeFormat=marshal encodeMayExecuteInput -# TODO: These tests should be merged with python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_string.py +# TODO: These tests should be merged with python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/test_string.py base64.b64encode(bs) # $ encodeInput=bs encodeOutput=base64.b64encode(..) encodeFormat=Base64 base64.standard_b64encode(bs) # $ encodeInput=bs encodeOutput=base64.standard_b64encode(..) encodeFormat=Base64 base64.urlsafe_b64encode(bs) # $ encodeInput=bs encodeOutput=base64.urlsafe_b64encode(..) encodeFormat=Base64 From bb00d6919aa6b8bbf01c5a097fac3d4cde4c7cc4 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Fri, 19 Apr 2024 10:47:25 +0200 Subject: [PATCH 10/39] Python: Move dataflow TestUtil to importable location --- .../TestUtil => TestUtilities/dataflow}/DataflowQueryTest.qll | 0 .../dataflow/TestUtil => TestUtilities/dataflow}/FlowTest.qll | 0 .../TestUtil => TestUtilities/dataflow}/LocalFlowStepTest.qll | 0 .../TestUtil => TestUtilities/dataflow}/MaximalFlowTest.qll | 0 .../dataflow}/NormalDataflowTest.qll | 4 ++-- .../dataflow}/NormalTaintTrackingTest.qll | 4 ++-- .../TestUtil => TestUtilities/dataflow}/RoutingTest.qll | 0 .../TestUtil => TestUtilities/dataflow}/UnresolvedCalls.qll | 0 .../dataflow/callGraphConfig.qll | 0 .../{library-tests => TestUtilities}/dataflow/testConfig.qll | 0 .../dataflow/testTaintConfig.qll | 0 python/ql/test/experimental/meta/debug/dataflowTestPaths.ql | 2 +- .../Security/CWE-022-UnsafeUnpacking/DataflowQueryTest.ql | 2 +- .../query-tests/Security/CWE-409/DataflowQueryTest.ql | 2 +- .../ql/test/library-tests/dataflow/basic/localFlowStepTest.ql | 2 +- .../ql/test/library-tests/dataflow/basic/maximalFlowTest.ql | 2 +- .../library-tests/dataflow/coverage/NormalDataflowTest.ql | 2 +- .../library-tests/dataflow/coverage/argumentRoutingTest.ql | 2 +- .../library-tests/dataflow/exceptions/NormalDataflowTest.ql | 2 +- .../library-tests/dataflow/fieldflow/NormalDataflowTest.ql | 2 +- .../test/library-tests/dataflow/fieldflow/UnresolvedCalls.ql | 2 +- .../test/library-tests/dataflow/match/NormalDataflowTest.ql | 2 +- .../dataflow/model-summaries/NormalDataflowTest.ql | 2 +- .../library-tests/dataflow/module-initialization/localFlow.ql | 2 +- python/ql/test/library-tests/dataflow/path-graph/PathNodes.ql | 2 +- python/ql/test/library-tests/dataflow/regression/dataflow.ql | 2 +- .../dataflow/summaries/NormalTaintTrackingTest.ql | 2 +- python/ql/test/library-tests/dataflow/summaries/summaries.ql | 2 +- .../tainttracking/generator-flow/NormalDataflowTest.ql | 2 +- .../library-tests/dataflow/variable-capture/CaptureTest.ql | 2 +- .../library-tests/frameworks/django-orm/NormalDataflowTest.ql | 2 +- .../Security/CWE-022-PathInjection/DataflowQueryTest.ql | 2 +- .../Security/CWE-078-CommandInjection/DataflowQueryTest.ql | 2 +- .../DataflowQueryTest.ql | 2 +- .../Security/CWE-943-NoSqlInjection/DataflowQueryTest.ql | 2 +- 35 files changed, 28 insertions(+), 28 deletions(-) rename python/ql/test/{library-tests/dataflow/TestUtil => TestUtilities/dataflow}/DataflowQueryTest.qll (100%) rename python/ql/test/{library-tests/dataflow/TestUtil => TestUtilities/dataflow}/FlowTest.qll (100%) rename python/ql/test/{library-tests/dataflow/TestUtil => TestUtilities/dataflow}/LocalFlowStepTest.qll (100%) rename python/ql/test/{library-tests/dataflow/TestUtil => TestUtilities/dataflow}/MaximalFlowTest.qll (100%) rename python/ql/test/{library-tests/dataflow/TestUtil => TestUtilities/dataflow}/NormalDataflowTest.qll (93%) rename python/ql/test/{library-tests/dataflow/TestUtil => TestUtilities/dataflow}/NormalTaintTrackingTest.qll (92%) rename python/ql/test/{library-tests/dataflow/TestUtil => TestUtilities/dataflow}/RoutingTest.qll (100%) rename python/ql/test/{library-tests/dataflow/TestUtil => TestUtilities/dataflow}/UnresolvedCalls.qll (100%) rename python/ql/test/{library-tests => TestUtilities}/dataflow/callGraphConfig.qll (100%) rename python/ql/test/{library-tests => TestUtilities}/dataflow/testConfig.qll (100%) rename python/ql/test/{library-tests => TestUtilities}/dataflow/testTaintConfig.qll (100%) diff --git a/python/ql/test/library-tests/dataflow/TestUtil/DataflowQueryTest.qll b/python/ql/test/TestUtilities/dataflow/DataflowQueryTest.qll similarity index 100% rename from python/ql/test/library-tests/dataflow/TestUtil/DataflowQueryTest.qll rename to python/ql/test/TestUtilities/dataflow/DataflowQueryTest.qll diff --git a/python/ql/test/library-tests/dataflow/TestUtil/FlowTest.qll b/python/ql/test/TestUtilities/dataflow/FlowTest.qll similarity index 100% rename from python/ql/test/library-tests/dataflow/TestUtil/FlowTest.qll rename to python/ql/test/TestUtilities/dataflow/FlowTest.qll diff --git a/python/ql/test/library-tests/dataflow/TestUtil/LocalFlowStepTest.qll b/python/ql/test/TestUtilities/dataflow/LocalFlowStepTest.qll similarity index 100% rename from python/ql/test/library-tests/dataflow/TestUtil/LocalFlowStepTest.qll rename to python/ql/test/TestUtilities/dataflow/LocalFlowStepTest.qll diff --git a/python/ql/test/library-tests/dataflow/TestUtil/MaximalFlowTest.qll b/python/ql/test/TestUtilities/dataflow/MaximalFlowTest.qll similarity index 100% rename from python/ql/test/library-tests/dataflow/TestUtil/MaximalFlowTest.qll rename to python/ql/test/TestUtilities/dataflow/MaximalFlowTest.qll diff --git a/python/ql/test/library-tests/dataflow/TestUtil/NormalDataflowTest.qll b/python/ql/test/TestUtilities/dataflow/NormalDataflowTest.qll similarity index 93% rename from python/ql/test/library-tests/dataflow/TestUtil/NormalDataflowTest.qll rename to python/ql/test/TestUtilities/dataflow/NormalDataflowTest.qll index 51516faac8a..b89738b100e 100644 --- a/python/ql/test/library-tests/dataflow/TestUtil/NormalDataflowTest.qll +++ b/python/ql/test/TestUtilities/dataflow/NormalDataflowTest.qll @@ -1,6 +1,6 @@ import python -import experimental.dataflow.TestUtil.FlowTest -import experimental.dataflow.testConfig +import TestUtilities.dataflow.FlowTest +import TestUtilities.dataflow.testConfig private import semmle.python.dataflow.new.internal.PrintNode module DataFlowTest implements FlowTestSig { diff --git a/python/ql/test/library-tests/dataflow/TestUtil/NormalTaintTrackingTest.qll b/python/ql/test/TestUtilities/dataflow/NormalTaintTrackingTest.qll similarity index 92% rename from python/ql/test/library-tests/dataflow/TestUtil/NormalTaintTrackingTest.qll rename to python/ql/test/TestUtilities/dataflow/NormalTaintTrackingTest.qll index 23262dfb3e5..e63e962df4d 100644 --- a/python/ql/test/library-tests/dataflow/TestUtil/NormalTaintTrackingTest.qll +++ b/python/ql/test/TestUtilities/dataflow/NormalTaintTrackingTest.qll @@ -1,6 +1,6 @@ import python -import experimental.dataflow.TestUtil.FlowTest -import experimental.dataflow.testTaintConfig +import TestUtilities.dataflow.FlowTest +import TestUtilities.dataflow.testTaintConfig private import semmle.python.dataflow.new.internal.PrintNode module DataFlowTest implements FlowTestSig { diff --git a/python/ql/test/library-tests/dataflow/TestUtil/RoutingTest.qll b/python/ql/test/TestUtilities/dataflow/RoutingTest.qll similarity index 100% rename from python/ql/test/library-tests/dataflow/TestUtil/RoutingTest.qll rename to python/ql/test/TestUtilities/dataflow/RoutingTest.qll diff --git a/python/ql/test/library-tests/dataflow/TestUtil/UnresolvedCalls.qll b/python/ql/test/TestUtilities/dataflow/UnresolvedCalls.qll similarity index 100% rename from python/ql/test/library-tests/dataflow/TestUtil/UnresolvedCalls.qll rename to python/ql/test/TestUtilities/dataflow/UnresolvedCalls.qll diff --git a/python/ql/test/library-tests/dataflow/callGraphConfig.qll b/python/ql/test/TestUtilities/dataflow/callGraphConfig.qll similarity index 100% rename from python/ql/test/library-tests/dataflow/callGraphConfig.qll rename to python/ql/test/TestUtilities/dataflow/callGraphConfig.qll diff --git a/python/ql/test/library-tests/dataflow/testConfig.qll b/python/ql/test/TestUtilities/dataflow/testConfig.qll similarity index 100% rename from python/ql/test/library-tests/dataflow/testConfig.qll rename to python/ql/test/TestUtilities/dataflow/testConfig.qll diff --git a/python/ql/test/library-tests/dataflow/testTaintConfig.qll b/python/ql/test/TestUtilities/dataflow/testTaintConfig.qll similarity index 100% rename from python/ql/test/library-tests/dataflow/testTaintConfig.qll rename to python/ql/test/TestUtilities/dataflow/testTaintConfig.qll diff --git a/python/ql/test/experimental/meta/debug/dataflowTestPaths.ql b/python/ql/test/experimental/meta/debug/dataflowTestPaths.ql index 78d783033f5..4c0ab898686 100644 --- a/python/ql/test/experimental/meta/debug/dataflowTestPaths.ql +++ b/python/ql/test/experimental/meta/debug/dataflowTestPaths.ql @@ -9,7 +9,7 @@ // 3. if necessary, look at partial paths by (un)commenting appropriate lines import python import semmle.python.dataflow.new.DataFlow -import experimental.dataflow.testConfig +import TestUtilities.dataflow.testConfig module Config implements DataFlow::ConfigSig { predicate isSource(DataFlow::Node source) { TestConfig::isSource(source) } diff --git a/python/ql/test/experimental/query-tests/Security/CWE-022-UnsafeUnpacking/DataflowQueryTest.ql b/python/ql/test/experimental/query-tests/Security/CWE-022-UnsafeUnpacking/DataflowQueryTest.ql index ed7d650f536..9cbf6dd6ad8 100644 --- a/python/ql/test/experimental/query-tests/Security/CWE-022-UnsafeUnpacking/DataflowQueryTest.ql +++ b/python/ql/test/experimental/query-tests/Security/CWE-022-UnsafeUnpacking/DataflowQueryTest.ql @@ -1,4 +1,4 @@ import python -import experimental.dataflow.TestUtil.DataflowQueryTest +import TestUtilities.dataflow.DataflowQueryTest import experimental.Security.UnsafeUnpackQuery import FromTaintTrackingConfig diff --git a/python/ql/test/experimental/query-tests/Security/CWE-409/DataflowQueryTest.ql b/python/ql/test/experimental/query-tests/Security/CWE-409/DataflowQueryTest.ql index 24a2c302b98..c1724c29d2c 100644 --- a/python/ql/test/experimental/query-tests/Security/CWE-409/DataflowQueryTest.ql +++ b/python/ql/test/experimental/query-tests/Security/CWE-409/DataflowQueryTest.ql @@ -1,4 +1,4 @@ import python -import experimental.dataflow.TestUtil.DataflowQueryTest +import TestUtilities.dataflow.DataflowQueryTest import experimental.semmle.python.security.DecompressionBomb import FromTaintTrackingConfig diff --git a/python/ql/test/library-tests/dataflow/basic/localFlowStepTest.ql b/python/ql/test/library-tests/dataflow/basic/localFlowStepTest.ql index 6dca0190156..881592eeaeb 100644 --- a/python/ql/test/library-tests/dataflow/basic/localFlowStepTest.ql +++ b/python/ql/test/library-tests/dataflow/basic/localFlowStepTest.ql @@ -1 +1 @@ -import experimental.dataflow.TestUtil.LocalFlowStepTest +import TestUtilities.dataflow.LocalFlowStepTest diff --git a/python/ql/test/library-tests/dataflow/basic/maximalFlowTest.ql b/python/ql/test/library-tests/dataflow/basic/maximalFlowTest.ql index 618dae382f1..64867eb89da 100644 --- a/python/ql/test/library-tests/dataflow/basic/maximalFlowTest.ql +++ b/python/ql/test/library-tests/dataflow/basic/maximalFlowTest.ql @@ -1 +1 @@ -import experimental.dataflow.TestUtil.MaximalFlowTest +import TestUtilities.dataflow.MaximalFlowTest diff --git a/python/ql/test/library-tests/dataflow/coverage/NormalDataflowTest.ql b/python/ql/test/library-tests/dataflow/coverage/NormalDataflowTest.ql index 3ee344d0b87..f7e55d12ded 100644 --- a/python/ql/test/library-tests/dataflow/coverage/NormalDataflowTest.ql +++ b/python/ql/test/library-tests/dataflow/coverage/NormalDataflowTest.ql @@ -1,2 +1,2 @@ import python -import experimental.dataflow.TestUtil.NormalDataflowTest +import TestUtilities.dataflow.NormalDataflowTest diff --git a/python/ql/test/library-tests/dataflow/coverage/argumentRoutingTest.ql b/python/ql/test/library-tests/dataflow/coverage/argumentRoutingTest.ql index 1a4e860f555..02b503824d4 100644 --- a/python/ql/test/library-tests/dataflow/coverage/argumentRoutingTest.ql +++ b/python/ql/test/library-tests/dataflow/coverage/argumentRoutingTest.ql @@ -1,7 +1,7 @@ import python import semmle.python.dataflow.new.DataFlow private import semmle.python.dataflow.new.internal.DataFlowPrivate as DataFlowPrivate -import experimental.dataflow.TestUtil.RoutingTest +import TestUtilities.dataflow.RoutingTest module Argument1RoutingTest implements RoutingTestSig { class Argument = Unit; diff --git a/python/ql/test/library-tests/dataflow/exceptions/NormalDataflowTest.ql b/python/ql/test/library-tests/dataflow/exceptions/NormalDataflowTest.ql index 3ee344d0b87..f7e55d12ded 100644 --- a/python/ql/test/library-tests/dataflow/exceptions/NormalDataflowTest.ql +++ b/python/ql/test/library-tests/dataflow/exceptions/NormalDataflowTest.ql @@ -1,2 +1,2 @@ import python -import experimental.dataflow.TestUtil.NormalDataflowTest +import TestUtilities.dataflow.NormalDataflowTest diff --git a/python/ql/test/library-tests/dataflow/fieldflow/NormalDataflowTest.ql b/python/ql/test/library-tests/dataflow/fieldflow/NormalDataflowTest.ql index 3ee344d0b87..f7e55d12ded 100644 --- a/python/ql/test/library-tests/dataflow/fieldflow/NormalDataflowTest.ql +++ b/python/ql/test/library-tests/dataflow/fieldflow/NormalDataflowTest.ql @@ -1,2 +1,2 @@ import python -import experimental.dataflow.TestUtil.NormalDataflowTest +import TestUtilities.dataflow.NormalDataflowTest diff --git a/python/ql/test/library-tests/dataflow/fieldflow/UnresolvedCalls.ql b/python/ql/test/library-tests/dataflow/fieldflow/UnresolvedCalls.ql index 3c7498bd651..299339dacf8 100644 --- a/python/ql/test/library-tests/dataflow/fieldflow/UnresolvedCalls.ql +++ b/python/ql/test/library-tests/dataflow/fieldflow/UnresolvedCalls.ql @@ -1,5 +1,5 @@ import python -import experimental.dataflow.TestUtil.UnresolvedCalls +import TestUtilities.dataflow.UnresolvedCalls private import semmle.python.dataflow.new.DataFlow module IgnoreDictMethod implements UnresolvedCallExpectationsSig { diff --git a/python/ql/test/library-tests/dataflow/match/NormalDataflowTest.ql b/python/ql/test/library-tests/dataflow/match/NormalDataflowTest.ql index 3ee344d0b87..f7e55d12ded 100644 --- a/python/ql/test/library-tests/dataflow/match/NormalDataflowTest.ql +++ b/python/ql/test/library-tests/dataflow/match/NormalDataflowTest.ql @@ -1,2 +1,2 @@ import python -import experimental.dataflow.TestUtil.NormalDataflowTest +import TestUtilities.dataflow.NormalDataflowTest diff --git a/python/ql/test/library-tests/dataflow/model-summaries/NormalDataflowTest.ql b/python/ql/test/library-tests/dataflow/model-summaries/NormalDataflowTest.ql index 3ee344d0b87..f7e55d12ded 100644 --- a/python/ql/test/library-tests/dataflow/model-summaries/NormalDataflowTest.ql +++ b/python/ql/test/library-tests/dataflow/model-summaries/NormalDataflowTest.ql @@ -1,2 +1,2 @@ import python -import experimental.dataflow.TestUtil.NormalDataflowTest +import TestUtilities.dataflow.NormalDataflowTest diff --git a/python/ql/test/library-tests/dataflow/module-initialization/localFlow.ql b/python/ql/test/library-tests/dataflow/module-initialization/localFlow.ql index 22a0f88d77e..36aa6e007a7 100644 --- a/python/ql/test/library-tests/dataflow/module-initialization/localFlow.ql +++ b/python/ql/test/library-tests/dataflow/module-initialization/localFlow.ql @@ -1,6 +1,6 @@ // This query should be more focused yet. import python -import experimental.dataflow.TestUtil.FlowTest +import TestUtilities.dataflow.FlowTest private import semmle.python.dataflow.new.internal.PrintNode private import semmle.python.dataflow.new.internal.DataFlowPrivate as DP diff --git a/python/ql/test/library-tests/dataflow/path-graph/PathNodes.ql b/python/ql/test/library-tests/dataflow/path-graph/PathNodes.ql index c403d588d49..8100e999491 100644 --- a/python/ql/test/library-tests/dataflow/path-graph/PathNodes.ql +++ b/python/ql/test/library-tests/dataflow/path-graph/PathNodes.ql @@ -5,7 +5,7 @@ import python import semmle.python.dataflow.new.DataFlow import semmle.python.dataflow.new.TaintTracking -import experimental.dataflow.testConfig +import TestUtilities.dataflow.testConfig import TestUtilities.InlineExpectationsTest module TestTaintFlow = TaintTracking::Global; diff --git a/python/ql/test/library-tests/dataflow/regression/dataflow.ql b/python/ql/test/library-tests/dataflow/regression/dataflow.ql index 39763fa4814..27645d084b8 100644 --- a/python/ql/test/library-tests/dataflow/regression/dataflow.ql +++ b/python/ql/test/library-tests/dataflow/regression/dataflow.ql @@ -6,7 +6,7 @@ */ import python -import experimental.dataflow.testConfig +import TestUtilities.dataflow.testConfig from DataFlow::Node source, DataFlow::Node sink where TestFlow::flow(source, sink) diff --git a/python/ql/test/library-tests/dataflow/summaries/NormalTaintTrackingTest.ql b/python/ql/test/library-tests/dataflow/summaries/NormalTaintTrackingTest.ql index afb44b6b2ed..59376d7a53c 100644 --- a/python/ql/test/library-tests/dataflow/summaries/NormalTaintTrackingTest.ql +++ b/python/ql/test/library-tests/dataflow/summaries/NormalTaintTrackingTest.ql @@ -1,3 +1,3 @@ import python private import TestSummaries -import experimental.dataflow.TestUtil.NormalTaintTrackingTest +import TestUtilities.dataflow.NormalTaintTrackingTest diff --git a/python/ql/test/library-tests/dataflow/summaries/summaries.ql b/python/ql/test/library-tests/dataflow/summaries/summaries.ql index e2a61cd6f46..4b5e79d2886 100644 --- a/python/ql/test/library-tests/dataflow/summaries/summaries.ql +++ b/python/ql/test/library-tests/dataflow/summaries/summaries.ql @@ -8,7 +8,7 @@ import TestFlow::PathGraph import semmle.python.dataflow.new.TaintTracking import semmle.python.dataflow.new.internal.FlowSummaryImpl import semmle.python.ApiGraphs -import experimental.dataflow.testTaintConfig +import TestUtilities.dataflow.testTaintConfig private import TestSummaries query predicate invalidSpecComponent(SummarizedCallable sc, string s, string c) { diff --git a/python/ql/test/library-tests/dataflow/tainttracking/generator-flow/NormalDataflowTest.ql b/python/ql/test/library-tests/dataflow/tainttracking/generator-flow/NormalDataflowTest.ql index 3ee344d0b87..f7e55d12ded 100644 --- a/python/ql/test/library-tests/dataflow/tainttracking/generator-flow/NormalDataflowTest.ql +++ b/python/ql/test/library-tests/dataflow/tainttracking/generator-flow/NormalDataflowTest.ql @@ -1,2 +1,2 @@ import python -import experimental.dataflow.TestUtil.NormalDataflowTest +import TestUtilities.dataflow.NormalDataflowTest diff --git a/python/ql/test/library-tests/dataflow/variable-capture/CaptureTest.ql b/python/ql/test/library-tests/dataflow/variable-capture/CaptureTest.ql index a1c754e8ee5..f8110174151 100644 --- a/python/ql/test/library-tests/dataflow/variable-capture/CaptureTest.ql +++ b/python/ql/test/library-tests/dataflow/variable-capture/CaptureTest.ql @@ -1,7 +1,7 @@ import python import semmle.python.dataflow.new.DataFlow import TestUtilities.InlineExpectationsTest -import experimental.dataflow.testConfig +import TestUtilities.dataflow.testConfig module CaptureTest implements TestSig { string getARelevantTag() { result = "captured" } diff --git a/python/ql/test/library-tests/frameworks/django-orm/NormalDataflowTest.ql b/python/ql/test/library-tests/frameworks/django-orm/NormalDataflowTest.ql index 3ee344d0b87..f7e55d12ded 100644 --- a/python/ql/test/library-tests/frameworks/django-orm/NormalDataflowTest.ql +++ b/python/ql/test/library-tests/frameworks/django-orm/NormalDataflowTest.ql @@ -1,2 +1,2 @@ import python -import experimental.dataflow.TestUtil.NormalDataflowTest +import TestUtilities.dataflow.NormalDataflowTest diff --git a/python/ql/test/query-tests/Security/CWE-022-PathInjection/DataflowQueryTest.ql b/python/ql/test/query-tests/Security/CWE-022-PathInjection/DataflowQueryTest.ql index 90761391ce7..a0cdc79b17d 100644 --- a/python/ql/test/query-tests/Security/CWE-022-PathInjection/DataflowQueryTest.ql +++ b/python/ql/test/query-tests/Security/CWE-022-PathInjection/DataflowQueryTest.ql @@ -1,4 +1,4 @@ import python -import experimental.dataflow.TestUtil.DataflowQueryTest +import TestUtilities.dataflow.DataflowQueryTest import semmle.python.security.dataflow.PathInjectionQuery import FromTaintTrackingStateConfig diff --git a/python/ql/test/query-tests/Security/CWE-078-CommandInjection/DataflowQueryTest.ql b/python/ql/test/query-tests/Security/CWE-078-CommandInjection/DataflowQueryTest.ql index df745b4ee8c..26350c3db65 100644 --- a/python/ql/test/query-tests/Security/CWE-078-CommandInjection/DataflowQueryTest.ql +++ b/python/ql/test/query-tests/Security/CWE-078-CommandInjection/DataflowQueryTest.ql @@ -1,4 +1,4 @@ import python -import experimental.dataflow.TestUtil.DataflowQueryTest +import TestUtilities.dataflow.DataflowQueryTest import semmle.python.security.dataflow.CommandInjectionQuery import FromTaintTrackingConfig diff --git a/python/ql/test/query-tests/Security/CWE-078-UnsafeShellCommandConstruction/DataflowQueryTest.ql b/python/ql/test/query-tests/Security/CWE-078-UnsafeShellCommandConstruction/DataflowQueryTest.ql index b2602b2b25c..521527e7e4f 100644 --- a/python/ql/test/query-tests/Security/CWE-078-UnsafeShellCommandConstruction/DataflowQueryTest.ql +++ b/python/ql/test/query-tests/Security/CWE-078-UnsafeShellCommandConstruction/DataflowQueryTest.ql @@ -1,4 +1,4 @@ import python -import experimental.dataflow.TestUtil.DataflowQueryTest +import TestUtilities.dataflow.DataflowQueryTest import semmle.python.security.dataflow.UnsafeShellCommandConstructionQuery import FromTaintTrackingConfig diff --git a/python/ql/test/query-tests/Security/CWE-943-NoSqlInjection/DataflowQueryTest.ql b/python/ql/test/query-tests/Security/CWE-943-NoSqlInjection/DataflowQueryTest.ql index b665aefd6fb..5123e883d9c 100644 --- a/python/ql/test/query-tests/Security/CWE-943-NoSqlInjection/DataflowQueryTest.ql +++ b/python/ql/test/query-tests/Security/CWE-943-NoSqlInjection/DataflowQueryTest.ql @@ -1,4 +1,4 @@ import python -import experimental.dataflow.TestUtil.DataflowQueryTest +import TestUtilities.dataflow.DataflowQueryTest import semmle.python.security.dataflow.NoSqlInjectionQuery import FromTaintTrackingStateConfig From 1bc085c8f74e836241b88f5cdf8962414bb7ec1f Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 23 Apr 2024 09:42:35 +0200 Subject: [PATCH 11/39] Python: Fixup for `callGraphConfig` --- python/ql/test/library-tests/dataflow/basic/callGraph.ql | 2 +- python/ql/test/library-tests/dataflow/basic/callGraphSinks.ql | 2 +- python/ql/test/library-tests/dataflow/basic/callGraphSources.ql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/ql/test/library-tests/dataflow/basic/callGraph.ql b/python/ql/test/library-tests/dataflow/basic/callGraph.ql index 98416af1d0f..f0e28c33dc6 100644 --- a/python/ql/test/library-tests/dataflow/basic/callGraph.ql +++ b/python/ql/test/library-tests/dataflow/basic/callGraph.ql @@ -1,4 +1,4 @@ -import experimental.dataflow.callGraphConfig +import TestUtilities.dataflow.callGraphConfig from DataFlow::Node source, DataFlow::Node sink where diff --git a/python/ql/test/library-tests/dataflow/basic/callGraphSinks.ql b/python/ql/test/library-tests/dataflow/basic/callGraphSinks.ql index 7cdf3cd94d1..472d6ccaa37 100644 --- a/python/ql/test/library-tests/dataflow/basic/callGraphSinks.ql +++ b/python/ql/test/library-tests/dataflow/basic/callGraphSinks.ql @@ -1,4 +1,4 @@ -import experimental.dataflow.callGraphConfig +import TestUtilities.dataflow.callGraphConfig from DataFlow::Node sink where diff --git a/python/ql/test/library-tests/dataflow/basic/callGraphSources.ql b/python/ql/test/library-tests/dataflow/basic/callGraphSources.ql index 8a0229c8f20..05b26caf3c0 100644 --- a/python/ql/test/library-tests/dataflow/basic/callGraphSources.ql +++ b/python/ql/test/library-tests/dataflow/basic/callGraphSources.ql @@ -1,4 +1,4 @@ -import experimental.dataflow.callGraphConfig +import TestUtilities.dataflow.callGraphConfig from DataFlow::Node source where From e4f23b31c69f36187f150015137bed94c94d3196 Mon Sep 17 00:00:00 2001 From: Asger F Date: Tue, 23 Apr 2024 20:04:08 +0200 Subject: [PATCH 12/39] JS: Add quotes around package name to correct parsing --- .../lib/semmle/javascript/frameworks/data/ModelsAsData.qll | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/javascript/ql/lib/semmle/javascript/frameworks/data/ModelsAsData.qll b/javascript/ql/lib/semmle/javascript/frameworks/data/ModelsAsData.qll index adfd683a497..fc3077d8986 100644 --- a/javascript/ql/lib/semmle/javascript/frameworks/data/ModelsAsData.qll +++ b/javascript/ql/lib/semmle/javascript/frameworks/data/ModelsAsData.qll @@ -108,7 +108,11 @@ module ModelExport { } predicate exposedName(API::Node node, string type, string path) { - node = API::moduleExport(type) and path = "" + exists(string moduleName | + node = API::moduleExport(moduleName) and + path = "" and + type = "'" + moduleName + "'" + ) } predicate suggestedName(API::Node node, string type) { From 9d00f660f18d5c7f46c90efae15a868e865706e1 Mon Sep 17 00:00:00 2001 From: Asger F Date: Tue, 23 Apr 2024 20:08:21 +0200 Subject: [PATCH 13/39] Update ModelGeneration.expected --- .../ModelGeneration/ModelGeneration.expected | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/javascript/ql/test/library-tests/ModelGeneration/ModelGeneration.expected b/javascript/ql/test/library-tests/ModelGeneration/ModelGeneration.expected index 26547b2814f..7a5cae4f796 100644 --- a/javascript/ql/test/library-tests/ModelGeneration/ModelGeneration.expected +++ b/javascript/ql/test/library-tests/ModelGeneration/ModelGeneration.expected @@ -1,17 +1,17 @@ typeModel -| (aliases).Alias1 | aliases | Member[Alias1] | -| (aliases).Alias1 | aliases | Member[Alias2] | -| (aliases).Alias1 | aliases | Member[Alias3].Member[x] | -| (aliases).Alias1 | aliases | Member[Alias4].Member[x].Member[x] | -| (aliases).Alias1 | aliases | Member[AliasedClass] | +| (aliases).Alias1 | 'aliases' | Member[Alias1] | +| (aliases).Alias1 | 'aliases' | Member[Alias2] | +| (aliases).Alias1 | 'aliases' | Member[Alias3].Member[x] | +| (aliases).Alias1 | 'aliases' | Member[Alias4].Member[x].Member[x] | +| (aliases).Alias1 | 'aliases' | Member[AliasedClass] | | (aliases).Alias1.prototype | (aliases).Alias1 | Instance | | (aliases).Alias1.prototype | (aliases).Alias1.prototype.foo | ReturnValue | | (aliases).Alias1.prototype.foo | (aliases).Alias1.prototype | Member[foo] | -| (long-access-path).a.shortcut.d | long-access-path | Member[a].Member[b].Member[c].Member[d] | -| (long-access-path).a.shortcut.d | long-access-path | Member[a].Member[shortcut].Member[d] | +| (long-access-path).a.shortcut.d | 'long-access-path' | Member[a].Member[b].Member[c].Member[d] | +| (long-access-path).a.shortcut.d | 'long-access-path' | Member[a].Member[shortcut].Member[d] | | (long-access-path).a.shortcut.d.e | (long-access-path).a.shortcut.d | Member[e] | -| (reexport).func | reexport | Member[func] | -| (return-this).FluentInterface | return-this | Member[FluentInterface] | +| (reexport).func | 'reexport' | Member[func] | +| (return-this).FluentInterface | 'return-this' | Member[FluentInterface] | | (return-this).FluentInterface.prototype | (return-this).FluentInterface | Instance | | (return-this).FluentInterface.prototype | (return-this).FluentInterface.prototype.bar | ReturnValue | | (return-this).FluentInterface.prototype | (return-this).FluentInterface.prototype.baz | ReturnValue | @@ -21,45 +21,45 @@ typeModel | (return-this).FluentInterface.prototype.foo | (return-this).FluentInterface.prototype | Member[foo] | | (return-this).FluentInterface.prototype.notFluent | (return-this).FluentInterface.prototype | Member[notFluent] | | (return-this).FluentInterface.prototype.notFluent2 | (return-this).FluentInterface.prototype | Member[notFluent2] | -| (root-function).PublicClass | root-function | Member[PublicClass] | +| (root-function).PublicClass | 'root-function' | Member[PublicClass] | +| (root-function).PublicClass.prototype | 'root-function' | ReturnValue | | (root-function).PublicClass.prototype | (root-function).PublicClass | Instance | -| (root-function).PublicClass.prototype | root-function | ReturnValue | | (root-function).PublicClass.prototype.method | (root-function).PublicClass.prototype | Member[method] | -| (semi-internal-class).PublicClass | semi-internal-class | Member[PublicClass] | +| (semi-internal-class).PublicClass | 'semi-internal-class' | Member[PublicClass] | | (semi-internal-class).PublicClass.prototype | (semi-internal-class).PublicClass | Instance | | (semi-internal-class).PublicClass.prototype | (semi-internal-class).SemiInternalClass.prototype.method | ReturnValue | | (semi-internal-class).PublicClass.prototype | (semi-internal-class).getAnonymous~expr2 | ReturnValue | | (semi-internal-class).PublicClass.prototype.publicMethod | (semi-internal-class).PublicClass.prototype | Member[publicMethod] | | (semi-internal-class).SemiInternalClass.prototype | (semi-internal-class).get | ReturnValue | | (semi-internal-class).SemiInternalClass.prototype.method | (semi-internal-class).SemiInternalClass.prototype | Member[method] | -| (semi-internal-class).get | semi-internal-class | Member[get] | -| (semi-internal-class).getAnonymous | semi-internal-class | Member[getAnonymous] | +| (semi-internal-class).get | 'semi-internal-class' | Member[get] | +| (semi-internal-class).getAnonymous | 'semi-internal-class' | Member[getAnonymous] | | (semi-internal-class).getAnonymous~expr1 | (semi-internal-class).getAnonymous | ReturnValue | | (semi-internal-class).getAnonymous~expr2 | (semi-internal-class).getAnonymous~expr1 | Member[method] | -| (subclass).A | subclass | Member[A] | +| (subclass).A | 'subclass' | Member[A] | | (subclass).A.prototype | (subclass).A | Instance | | (subclass).A.prototype | (subclass).B.prototype | | | (subclass).A.prototype | (subclass).ExposedMidSubClass.prototype~expr1 | | | (subclass).A.prototype.a | (subclass).A.prototype | Member[a] | -| (subclass).B | subclass | Member[B] | +| (subclass).B | 'subclass' | Member[B] | | (subclass).B.prototype | (subclass).B | Instance | | (subclass).B.prototype | (subclass).C.prototype | | | (subclass).B.prototype.b | (subclass).B.prototype | Member[b] | -| (subclass).C | subclass | Member[C] | +| (subclass).C | 'subclass' | Member[C] | | (subclass).C.prototype | (subclass).C | Instance | | (subclass).C.prototype.c | (subclass).C.prototype | Member[c] | -| (subclass).D | subclass | Member[D] | +| (subclass).D | 'subclass' | Member[D] | | (subclass).D.prototype | (subclass).D | Instance | | (subclass).D.prototype.d | (subclass).D.prototype | Member[d] | -| (subclass).ExposedMidSubClass | subclass | Member[ExposedMidSubClass] | +| (subclass).ExposedMidSubClass | 'subclass' | Member[ExposedMidSubClass] | | (subclass).ExposedMidSubClass.prototype | (subclass).ExposedMidSubClass | Instance | | (subclass).ExposedMidSubClass.prototype.m | (subclass).ExposedMidSubClass.prototype | Member[m] | | (subclass).ExposedMidSubClass.prototype~expr1 | (subclass).ExposedMidSubClass.prototype | | +| upstream-lib | 'reexport' | Member[lib] | | upstream-lib | (reexport).func | ReturnValue | -| upstream-lib | reexport | Member[lib] | | upstream-lib.Type | (subclass).D.prototype | | -| upstream-lib.XYZ | reexport | Member[x].Member[y].Member[z] | -| upstream-lib.XYZ | reexport | Member[xy].Member[z] | +| upstream-lib.XYZ | 'reexport' | Member[x].Member[y].Member[z] | +| upstream-lib.XYZ | 'reexport' | Member[xy].Member[z] | summaryModel | (aliases).Alias1.prototype | | | Member[foo].ReturnValue | type | | (return-this).FluentInterface.prototype | | | Member[bar].ReturnValue | type | From db07c162e43e6a24a9fde8bab631cf05135f5c07 Mon Sep 17 00:00:00 2001 From: Asger F Date: Tue, 23 Apr 2024 20:25:55 +0200 Subject: [PATCH 14/39] JS: Allow generated models to use (package) --- .../frameworks/data/ModelsAsData.qll | 2 +- .../data/internal/ApiGraphModelsSpecific.qll | 8 ++++ .../ModelGeneration/ModelGeneration.expected | 44 +++++++++---------- .../frameworks/data/test.expected | 1 + .../frameworks/data/test.ext.yml | 3 +- .../library-tests/frameworks/data/test.js | 2 + 6 files changed, 36 insertions(+), 24 deletions(-) diff --git a/javascript/ql/lib/semmle/javascript/frameworks/data/ModelsAsData.qll b/javascript/ql/lib/semmle/javascript/frameworks/data/ModelsAsData.qll index fc3077d8986..6e95955749b 100644 --- a/javascript/ql/lib/semmle/javascript/frameworks/data/ModelsAsData.qll +++ b/javascript/ql/lib/semmle/javascript/frameworks/data/ModelsAsData.qll @@ -111,7 +111,7 @@ module ModelExport { exists(string moduleName | node = API::moduleExport(moduleName) and path = "" and - type = "'" + moduleName + "'" + type = "(" + moduleName + ")" ) } diff --git a/javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModelsSpecific.qll b/javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModelsSpecific.qll index 1a7b7e9bf32..1b616a199bc 100644 --- a/javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModelsSpecific.qll +++ b/javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModelsSpecific.qll @@ -34,6 +34,11 @@ class Location = JS::Location; * * Type names have form `package.type` or just `package` if referring to the package export * object. If `package` contains a `.` character it must be enclosed in single quotes, such as `'package'.type`. + * + * A type name of form `(package)` may also be used when refering to the package export object. + * We allow this syntax as an alternative to the above, so models generated based on `EndpointNaming` look more consistent. + * However, access paths are deliberately not parsed here, as we can not handle aliasing at this stage. + * The model generator must explicitly generate the step between `(package)` and `(package).foo`, for example. */ bindingset[rawType] predicate parseTypeString(string rawType, string package, string qualifiedName) { @@ -42,6 +47,9 @@ predicate parseTypeString(string rawType, string package, string qualifiedName) package = rawType.regexpCapture(regexp, 1).regexpReplaceAll("^'|'$", "") and qualifiedName = rawType.regexpCapture(regexp, 2).regexpReplaceAll("^\\.", "") ) + or + package = rawType.regexpCapture("[(]([^)]+)[)]", 1) and + qualifiedName = "" } /** diff --git a/javascript/ql/test/library-tests/ModelGeneration/ModelGeneration.expected b/javascript/ql/test/library-tests/ModelGeneration/ModelGeneration.expected index 7a5cae4f796..d983b77b538 100644 --- a/javascript/ql/test/library-tests/ModelGeneration/ModelGeneration.expected +++ b/javascript/ql/test/library-tests/ModelGeneration/ModelGeneration.expected @@ -1,17 +1,17 @@ typeModel -| (aliases).Alias1 | 'aliases' | Member[Alias1] | -| (aliases).Alias1 | 'aliases' | Member[Alias2] | -| (aliases).Alias1 | 'aliases' | Member[Alias3].Member[x] | -| (aliases).Alias1 | 'aliases' | Member[Alias4].Member[x].Member[x] | -| (aliases).Alias1 | 'aliases' | Member[AliasedClass] | +| (aliases).Alias1 | (aliases) | Member[Alias1] | +| (aliases).Alias1 | (aliases) | Member[Alias2] | +| (aliases).Alias1 | (aliases) | Member[Alias3].Member[x] | +| (aliases).Alias1 | (aliases) | Member[Alias4].Member[x].Member[x] | +| (aliases).Alias1 | (aliases) | Member[AliasedClass] | | (aliases).Alias1.prototype | (aliases).Alias1 | Instance | | (aliases).Alias1.prototype | (aliases).Alias1.prototype.foo | ReturnValue | | (aliases).Alias1.prototype.foo | (aliases).Alias1.prototype | Member[foo] | -| (long-access-path).a.shortcut.d | 'long-access-path' | Member[a].Member[b].Member[c].Member[d] | -| (long-access-path).a.shortcut.d | 'long-access-path' | Member[a].Member[shortcut].Member[d] | +| (long-access-path).a.shortcut.d | (long-access-path) | Member[a].Member[b].Member[c].Member[d] | +| (long-access-path).a.shortcut.d | (long-access-path) | Member[a].Member[shortcut].Member[d] | | (long-access-path).a.shortcut.d.e | (long-access-path).a.shortcut.d | Member[e] | -| (reexport).func | 'reexport' | Member[func] | -| (return-this).FluentInterface | 'return-this' | Member[FluentInterface] | +| (reexport).func | (reexport) | Member[func] | +| (return-this).FluentInterface | (return-this) | Member[FluentInterface] | | (return-this).FluentInterface.prototype | (return-this).FluentInterface | Instance | | (return-this).FluentInterface.prototype | (return-this).FluentInterface.prototype.bar | ReturnValue | | (return-this).FluentInterface.prototype | (return-this).FluentInterface.prototype.baz | ReturnValue | @@ -21,45 +21,45 @@ typeModel | (return-this).FluentInterface.prototype.foo | (return-this).FluentInterface.prototype | Member[foo] | | (return-this).FluentInterface.prototype.notFluent | (return-this).FluentInterface.prototype | Member[notFluent] | | (return-this).FluentInterface.prototype.notFluent2 | (return-this).FluentInterface.prototype | Member[notFluent2] | -| (root-function).PublicClass | 'root-function' | Member[PublicClass] | -| (root-function).PublicClass.prototype | 'root-function' | ReturnValue | +| (root-function).PublicClass | (root-function) | Member[PublicClass] | +| (root-function).PublicClass.prototype | (root-function) | ReturnValue | | (root-function).PublicClass.prototype | (root-function).PublicClass | Instance | | (root-function).PublicClass.prototype.method | (root-function).PublicClass.prototype | Member[method] | -| (semi-internal-class).PublicClass | 'semi-internal-class' | Member[PublicClass] | +| (semi-internal-class).PublicClass | (semi-internal-class) | Member[PublicClass] | | (semi-internal-class).PublicClass.prototype | (semi-internal-class).PublicClass | Instance | | (semi-internal-class).PublicClass.prototype | (semi-internal-class).SemiInternalClass.prototype.method | ReturnValue | | (semi-internal-class).PublicClass.prototype | (semi-internal-class).getAnonymous~expr2 | ReturnValue | | (semi-internal-class).PublicClass.prototype.publicMethod | (semi-internal-class).PublicClass.prototype | Member[publicMethod] | | (semi-internal-class).SemiInternalClass.prototype | (semi-internal-class).get | ReturnValue | | (semi-internal-class).SemiInternalClass.prototype.method | (semi-internal-class).SemiInternalClass.prototype | Member[method] | -| (semi-internal-class).get | 'semi-internal-class' | Member[get] | -| (semi-internal-class).getAnonymous | 'semi-internal-class' | Member[getAnonymous] | +| (semi-internal-class).get | (semi-internal-class) | Member[get] | +| (semi-internal-class).getAnonymous | (semi-internal-class) | Member[getAnonymous] | | (semi-internal-class).getAnonymous~expr1 | (semi-internal-class).getAnonymous | ReturnValue | | (semi-internal-class).getAnonymous~expr2 | (semi-internal-class).getAnonymous~expr1 | Member[method] | -| (subclass).A | 'subclass' | Member[A] | +| (subclass).A | (subclass) | Member[A] | | (subclass).A.prototype | (subclass).A | Instance | | (subclass).A.prototype | (subclass).B.prototype | | | (subclass).A.prototype | (subclass).ExposedMidSubClass.prototype~expr1 | | | (subclass).A.prototype.a | (subclass).A.prototype | Member[a] | -| (subclass).B | 'subclass' | Member[B] | +| (subclass).B | (subclass) | Member[B] | | (subclass).B.prototype | (subclass).B | Instance | | (subclass).B.prototype | (subclass).C.prototype | | | (subclass).B.prototype.b | (subclass).B.prototype | Member[b] | -| (subclass).C | 'subclass' | Member[C] | +| (subclass).C | (subclass) | Member[C] | | (subclass).C.prototype | (subclass).C | Instance | | (subclass).C.prototype.c | (subclass).C.prototype | Member[c] | -| (subclass).D | 'subclass' | Member[D] | +| (subclass).D | (subclass) | Member[D] | | (subclass).D.prototype | (subclass).D | Instance | | (subclass).D.prototype.d | (subclass).D.prototype | Member[d] | -| (subclass).ExposedMidSubClass | 'subclass' | Member[ExposedMidSubClass] | +| (subclass).ExposedMidSubClass | (subclass) | Member[ExposedMidSubClass] | | (subclass).ExposedMidSubClass.prototype | (subclass).ExposedMidSubClass | Instance | | (subclass).ExposedMidSubClass.prototype.m | (subclass).ExposedMidSubClass.prototype | Member[m] | | (subclass).ExposedMidSubClass.prototype~expr1 | (subclass).ExposedMidSubClass.prototype | | -| upstream-lib | 'reexport' | Member[lib] | +| upstream-lib | (reexport) | Member[lib] | | upstream-lib | (reexport).func | ReturnValue | | upstream-lib.Type | (subclass).D.prototype | | -| upstream-lib.XYZ | 'reexport' | Member[x].Member[y].Member[z] | -| upstream-lib.XYZ | 'reexport' | Member[xy].Member[z] | +| upstream-lib.XYZ | (reexport) | Member[x].Member[y].Member[z] | +| upstream-lib.XYZ | (reexport) | Member[xy].Member[z] | summaryModel | (aliases).Alias1.prototype | | | Member[foo].ReturnValue | type | | (return-this).FluentInterface.prototype | | | Member[bar].ReturnValue | type | diff --git a/javascript/ql/test/library-tests/frameworks/data/test.expected b/javascript/ql/test/library-tests/frameworks/data/test.expected index 843b1f32d5b..d9b9d7e18e2 100644 --- a/javascript/ql/test/library-tests/frameworks/data/test.expected +++ b/javascript/ql/test/library-tests/frameworks/data/test.expected @@ -78,6 +78,7 @@ taintFlow | test.js:265:6:265:39 | new MyS ... ource() | test.js:265:6:265:39 | new MyS ... ource() | | test.js:269:10:269:31 | this.ba ... ource() | test.js:269:10:269:31 | this.ba ... ource() | | test.js:272:6:272:40 | new MyS ... ource() | test.js:272:6:272:40 | new MyS ... ource() | +| test.js:274:6:274:39 | testlib ... eName() | test.js:274:6:274:39 | testlib ... eName() | isSink | test.js:54:18:54:25 | source() | test-sink | | test.js:55:22:55:29 | source() | test-sink | diff --git a/javascript/ql/test/library-tests/frameworks/data/test.ext.yml b/javascript/ql/test/library-tests/frameworks/data/test.ext.yml index 2387cc03978..8aec6766913 100644 --- a/javascript/ql/test/library-tests/frameworks/data/test.ext.yml +++ b/javascript/ql/test/library-tests/frameworks/data/test.ext.yml @@ -10,6 +10,7 @@ extensions: - ['testlib', 'Member[MethodDecorator].DecoratedMember.Parameter[0]', 'test-source'] - ['testlib', 'Member[ParamDecoratorSource].DecoratedParameter', 'test-source'] - ['testlib', 'Member[getSource].ReturnValue', 'test-source'] + - ['(testlib)', 'Member[parenthesizedPackageName].ReturnValue', 'test-source'] - addsTo: pack: codeql/javascript-all @@ -73,4 +74,4 @@ extensions: data: - ['ABC', 'Member[a].Member[b].WithArity[0].ReturnValue.Member[c]'] - ['LeftRight', 'Member[left].TypeVar[LeftRight].Member[right]'] - - ['LeftRight', 'Member[x]'] \ No newline at end of file + - ['LeftRight', 'Member[x]'] diff --git a/javascript/ql/test/library-tests/frameworks/data/test.js b/javascript/ql/test/library-tests/frameworks/data/test.js index bbcb10418a1..97bb49f8cf7 100644 --- a/javascript/ql/test/library-tests/frameworks/data/test.js +++ b/javascript/ql/test/library-tests/frameworks/data/test.js @@ -270,3 +270,5 @@ class MySubclass2 extends MySubclass { } } sink(new MySubclass2().baseclassSource()); // NOT OK + +sink(testlib.parenthesizedPackageName()); // NOT OK From 88e67715a143323a1bfde1c985d0c18ee84b7a48 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Wed, 24 Apr 2024 11:53:26 +0200 Subject: [PATCH 15/39] C#: Do not download `Microsoft.CodeAnalysis.ResxSourceGenerator` when there are no `resx` files to process --- .../SourceGenerators/ResxGenerator.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/ResxGenerator.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/ResxGenerator.cs index 3c99350a9de..ff24bf0ea6f 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/ResxGenerator.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/ResxGenerator.cs @@ -19,6 +19,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching TemporaryDirectory tempWorkingDirectory, IEnumerable references) : base(fileProvider, fileContent, dotnet, compilationInfoContainer, logger, tempWorkingDirectory, references) { + if (fileProvider.Resources.Count == 0) + { + logger.LogDebug("No resources found, skipping resource extraction."); + sourceGeneratorFolder = null; + return; + } + try { // The package is downloaded to `missingpackages`, which is okay, we're already after the DLL collection phase. From f3daba510bd772cbc878212ae12ef123d4db134e Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Wed, 24 Apr 2024 11:57:34 +0200 Subject: [PATCH 16/39] C#: Fix `global.json` and `packages.config` lookup --- .../Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs | 2 +- .../Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs index c573c5ff4e6..74c1a37154d 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs @@ -143,7 +143,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching // See https://docs.microsoft.com/en-us/dotnet/core/tools/global-json var versions = new List(); - foreach (var path in files.Where(p => p.EndsWith("global.json", StringComparison.Ordinal))) + foreach (var path in files.Where(p => p.EndsWith(Path.DirectorySeparatorChar + "global.json", StringComparison.Ordinal))) { try { diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs index 44cff0a8085..467289c7e0d 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs @@ -184,7 +184,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { try { - var isPackagesConfig = file.EndsWith("packages.config", StringComparison.OrdinalIgnoreCase); + var isPackagesConfig = file.EndsWith(Path.DirectorySeparatorChar + "packages.config", StringComparison.OrdinalIgnoreCase); foreach (ReadOnlySpan line in unsafeFileReader.ReadLines(file)) { From 95d579d9de7542ff5d31b60ca40445cbf28f71cb Mon Sep 17 00:00:00 2001 From: Tom Hvitved Date: Wed, 24 Apr 2024 12:22:11 +0200 Subject: [PATCH 17/39] Data flow: Fix bad join ``` Evaluated relational algebra for predicate _DataFlowImpl::Impl::ret__#count_range@d112335l with tuple counts: 285176 ~2% {3} r1 = SCAN `_DataFlowDispatch::DataFlowCall.getEnclosingCallable/0#dispred#b7b78b19_DataFlowImpl::Impl::returnCallEdge1/4#d02cae42_2301#join_rhs` ON FIRST 2 OUTPUT Lhs.0, Lhs.2, Rhs.2, Lhs.1, Rhs.3 39070 ~8% {6} | JOIN WITH `DataFlowImplCommon::Cached::viableImplInCallContextExt/2#58e931ad` ON FIRST 3 OUTPUT Lhs.0, Lhs.3, Lhs.1, Lhs.2, Lhs.4, _ 39070 ~0% {6} | REWRITE WITH Out.5 := 1 return r1 ``` --- .../codeql/dataflow/internal/DataFlowImpl.qll | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/shared/dataflow/codeql/dataflow/internal/DataFlowImpl.qll b/shared/dataflow/codeql/dataflow/internal/DataFlowImpl.qll index 9dfbc3e0bcd..de9f718b347 100644 --- a/shared/dataflow/codeql/dataflow/internal/DataFlowImpl.qll +++ b/shared/dataflow/codeql/dataflow/internal/DataFlowImpl.qll @@ -1142,6 +1142,14 @@ module MakeImpl Lang> { ) } + pragma[nomagic] + private predicate returnCallEdgeInCtx1( + DataFlowCallable c, SndLevelScopeOption scope, DataFlowCall call, NodeEx out, DataFlowCall ctx + ) { + returnCallEdge1(c, scope, call, out) and + c = viableImplInCallContextExt(call, ctx) + } + private int ctxDispatchFanoutOnReturn(NodeEx out, DataFlowCall ctx) { exists(DataFlowCall call, DataFlowCallable c | simpleDispatchFanoutOnReturn(call, out) > 1 and @@ -1151,8 +1159,7 @@ module MakeImpl Lang> { mayBenefitFromCallContextExt(call, _) and result = count(DataFlowCallable tgt, SndLevelScopeOption scope | - tgt = viableImplInCallContextExt(call, ctx) and - returnCallEdge1(tgt, scope, call, out) + returnCallEdgeInCtx1(tgt, scope, call, out, ctx) ) ) } From 8f2e51faa6641905bc29dfa28bc5df09fd575c1f Mon Sep 17 00:00:00 2001 From: Nick Rolfe Date: Wed, 24 Apr 2024 12:32:49 +0100 Subject: [PATCH 18/39] Ruby: do fewer regexp matches in SensitiveActions --- .../codeql/ruby/security/SensitiveActions.qll | 55 +++++++++++++------ 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/ruby/ql/lib/codeql/ruby/security/SensitiveActions.qll b/ruby/ql/lib/codeql/ruby/security/SensitiveActions.qll index cc01ab4078b..34beb33604b 100644 --- a/ruby/ql/lib/codeql/ruby/security/SensitiveActions.qll +++ b/ruby/ql/lib/codeql/ruby/security/SensitiveActions.qll @@ -28,17 +28,13 @@ abstract class SensitiveNode extends DataFlow::Node { } /** A method call that might produce sensitive data. */ -class SensitiveCall extends SensitiveNode instanceof DataFlow::CallNode { +abstract class SensitiveCall extends SensitiveNode { } + +private class SensitiveDataMethodNameCall extends SensitiveCall instanceof DataFlow::CallNode { SensitiveDataClassification classification; - SensitiveCall() { + SensitiveDataMethodNameCall() { classification = this.getMethodName().(SensitiveDataMethodName).getClassification() - or - // This is particularly to pick up methods with an argument like "password", which - // may indicate a lookup. - exists(string s | super.getArgument(_).asExpr().getConstantValue().isStringlikeValue(s) | - nameIndicatesSensitiveData(s, classification) - ) } override string describe() { result = "a call to " + super.getMethodName() } @@ -46,6 +42,23 @@ class SensitiveCall extends SensitiveNode instanceof DataFlow::CallNode { override SensitiveDataClassification getClassification() { result = classification } } +private class SensitiveArgumentCall extends SensitiveCall instanceof DataFlow::CallNode { + string argName; + + SensitiveArgumentCall() { + // This is particularly to pick up methods with an argument like "password", which may indicate + // a lookup. + super.getArgument(_).asExpr().getConstantValue().isStringlikeValue(argName) and + nameIndicatesSensitiveData(argName) + } + + override string describe() { result = "a call to " + super.getMethodName() } + + override SensitiveDataClassification getClassification() { + nameIndicatesSensitiveData(argName, result) + } +} + /** An access to a variable or hash value that might contain sensitive data. */ abstract class SensitiveVariableAccess extends SensitiveNode { string name; @@ -93,7 +106,7 @@ private string unprefixedVariableName(string name) { result = name.regexpReplace /** A write to a variable or property that might contain sensitive data. */ private class BasicSensitiveWrite extends SensitiveWrite { - SensitiveDataClassification classification; + string unprefixedName; BasicSensitiveWrite() { exists(string name | @@ -111,23 +124,29 @@ private class BasicSensitiveWrite extends SensitiveWrite { */ writesProperty(this, name) and - nameIndicatesSensitiveData(unprefixedVariableName(name), classification) + unprefixedName = unprefixedVariableName(name) and + nameIndicatesSensitiveData(unprefixedName) ) } /** Gets a classification of the kind of sensitive data the write might handle. */ - SensitiveDataClassification getClassification() { result = classification } + SensitiveDataClassification getClassification() { + nameIndicatesSensitiveData(unprefixedName, result) + } } /** An access to a variable or hash value that might contain sensitive data. */ private class BasicSensitiveVariableAccess extends SensitiveVariableAccess { - SensitiveDataClassification classification; + string unprefixedName; BasicSensitiveVariableAccess() { - nameIndicatesSensitiveData(unprefixedVariableName(name), classification) + unprefixedName = unprefixedVariableName(name) and + nameIndicatesSensitiveData(unprefixedName) } - override SensitiveDataClassification getClassification() { result = classification } + override SensitiveDataClassification getClassification() { + nameIndicatesSensitiveData(unprefixedName, result) + } } /** A method name that suggests it may be sensitive. */ @@ -143,11 +162,11 @@ abstract class SensitiveDataMethodName extends SensitiveMethodName { /** A method name that might return sensitive credential data. */ class CredentialsMethodName extends SensitiveDataMethodName { - SensitiveDataClassification classification; + CredentialsMethodName() { nameIndicatesSensitiveData(this) } - CredentialsMethodName() { nameIndicatesSensitiveData(this, classification) } - - override SensitiveDataClassification getClassification() { result = classification } + override SensitiveDataClassification getClassification() { + nameIndicatesSensitiveData(this, result) + } } /** From 4a97f9589017937fd4f40f9a9a2d7c76e1b01452 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Wed, 24 Apr 2024 13:47:25 +0200 Subject: [PATCH 19/39] Improve code quality --- .../DotNet.cs | 2 +- .../FileContent.cs | 2 +- .../DotnetSourceGeneratorBase.cs | 26 +------------ csharp/extractor/Semmle.Util/FileUtils.cs | 37 +++++++++++++++++++ 4 files changed, 40 insertions(+), 27 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs index 74c1a37154d..95e8fe0675f 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs @@ -143,7 +143,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching // See https://docs.microsoft.com/en-us/dotnet/core/tools/global-json var versions = new List(); - foreach (var path in files.Where(p => p.EndsWith(Path.DirectorySeparatorChar + "global.json", StringComparison.Ordinal))) + foreach (var path in files.Where(p => string.Equals(FileUtils.SafeGetFileName(p, logger), "global.json", StringComparison.OrdinalIgnoreCase))) { try { diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs index 467289c7e0d..f33329046cf 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs @@ -184,7 +184,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { try { - var isPackagesConfig = file.EndsWith(Path.DirectorySeparatorChar + "packages.config", StringComparison.OrdinalIgnoreCase); + var isPackagesConfig = string.Equals(FileUtils.SafeGetFileName(file, logger), "packages.config", StringComparison.OrdinalIgnoreCase); foreach (ReadOnlySpan line in unsafeFileReader.ReadLines(file)) { diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorBase.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorBase.cs index 4d353ffbeea..461590348df 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorBase.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorBase.cs @@ -55,7 +55,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching // group additional files by closes project file: var projects = fileProvider.Projects - .Select(p => (File: p, Directory: SafeGetDirectoryName(p))) + .Select(p => (File: p, Directory: FileUtils.SafeGetDirectoryName(p, logger))) .Where(p => p.Directory.Length > 0); var groupedFiles = new Dictionary>(); @@ -93,30 +93,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - private string SafeGetDirectoryName(string fileName) - { - try - { - var dir = Path.GetDirectoryName(fileName); - if (dir is null) - { - return ""; - } - - if (!dir.EndsWith(Path.DirectorySeparatorChar)) - { - dir += Path.DirectorySeparatorChar; - } - - return dir; - } - catch (Exception ex) - { - logger.LogDebug($"Failed to get directory name for {fileName}: {ex.Message}"); - return ""; - } - } - protected abstract ICollection AdditionalFiles { get; } protected abstract string FileType { get; } diff --git a/csharp/extractor/Semmle.Util/FileUtils.cs b/csharp/extractor/Semmle.Util/FileUtils.cs index 4a22877e3c1..4d9052bcc4e 100644 --- a/csharp/extractor/Semmle.Util/FileUtils.cs +++ b/csharp/extractor/Semmle.Util/FileUtils.cs @@ -185,5 +185,42 @@ namespace Semmle.Util return new FileInfo(outputPath); } + + public static string SafeGetDirectoryName(string path, ILogger logger) + { + try + { + var dir = Path.GetDirectoryName(path); + if (dir is null) + { + return ""; + } + + if (!dir.EndsWith(Path.DirectorySeparatorChar)) + { + dir += Path.DirectorySeparatorChar; + } + + return dir; + } + catch (Exception ex) + { + logger.LogDebug($"Failed to get directory name for {path}: {ex.Message}"); + return ""; + } + } + + public static string? SafeGetFileName(string path, ILogger logger) + { + try + { + return Path.GetFileName(path); + } + catch (Exception ex) + { + logger.LogDebug($"Failed to get file name for {path}: {ex.Message}"); + return null; + } + } } } From 3ef7a0932ad9249539bb90dc3f4f1e3bb46fae6e Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Wed, 20 Mar 2024 10:50:59 +0000 Subject: [PATCH 20/39] Add flow through string concatenation --- go/ql/src/Security/CWE-020/IncompleteHostnameRegexp.ql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/go/ql/src/Security/CWE-020/IncompleteHostnameRegexp.ql b/go/ql/src/Security/CWE-020/IncompleteHostnameRegexp.ql index 48731179127..0392f09e801 100644 --- a/go/ql/src/Security/CWE-020/IncompleteHostnameRegexp.ql +++ b/go/ql/src/Security/CWE-020/IncompleteHostnameRegexp.ql @@ -101,6 +101,10 @@ module IncompleteHostNameRegexpConfig implements DataFlow::ConfigSig { ) and not regexpGuardsError(sink) } + + predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) { + StringOps::Concatenation::taintStep(node1, node2) + } } module Flow = DataFlow::Global; From 0000c72329e778f15542c3764cdad8b6f3a8beb1 Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Wed, 20 Mar 2024 11:01:50 +0000 Subject: [PATCH 21/39] Remove attempt at avoiding duplicate alerts --- go/ql/src/Security/CWE-020/IncompleteHostnameRegexp.ql | 5 ----- 1 file changed, 5 deletions(-) diff --git a/go/ql/src/Security/CWE-020/IncompleteHostnameRegexp.ql b/go/ql/src/Security/CWE-020/IncompleteHostnameRegexp.ql index 0392f09e801..e114d924475 100644 --- a/go/ql/src/Security/CWE-020/IncompleteHostnameRegexp.ql +++ b/go/ql/src/Security/CWE-020/IncompleteHostnameRegexp.ql @@ -84,11 +84,6 @@ module IncompleteHostNameRegexpConfig implements DataFlow::ConfigSig { exists(Expr e | e = source.asExpr() and isIncompleteHostNameRegexpPattern(e.getStringValue(), hostPart) - | - e instanceof StringLit - or - e instanceof AddExpr and - not isIncompleteHostNameRegexpPattern(e.(AddExpr).getAnOperand().getStringValue(), _) ) } From 89623072916531254942b03cda2491539f178bb1 Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Tue, 23 Apr 2024 21:13:55 +0100 Subject: [PATCH 22/39] Add second good go file to tests --- .../CWE-020/IncompleteHostnameRegexpGood2.go | 2 +- .../IncompleteHostnameRegexpGood2.go | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 go/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegexp/IncompleteHostnameRegexpGood2.go diff --git a/go/ql/src/Security/CWE-020/IncompleteHostnameRegexpGood2.go b/go/ql/src/Security/CWE-020/IncompleteHostnameRegexpGood2.go index 7c5df3f6742..c6c3fc06981 100644 --- a/go/ql/src/Security/CWE-020/IncompleteHostnameRegexpGood2.go +++ b/go/ql/src/Security/CWE-020/IncompleteHostnameRegexpGood2.go @@ -6,7 +6,7 @@ import ( "regexp" ) -func checkRedirectGood(req *http.Request, via []*http.Request) error { +func checkRedirectGood2(req *http.Request, via []*http.Request) error { // GOOD: the host of `req.URL` must be `example.com`, `www.example.com` or `beta.example.com` re := `^((www|beta)\.)?example\.com/` if matched, _ := regexp.MatchString(re, req.URL.Host); matched { diff --git a/go/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegexp/IncompleteHostnameRegexpGood2.go b/go/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegexp/IncompleteHostnameRegexpGood2.go new file mode 100644 index 00000000000..c6c3fc06981 --- /dev/null +++ b/go/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegexp/IncompleteHostnameRegexpGood2.go @@ -0,0 +1,16 @@ +package main + +import ( + "errors" + "net/http" + "regexp" +) + +func checkRedirectGood2(req *http.Request, via []*http.Request) error { + // GOOD: the host of `req.URL` must be `example.com`, `www.example.com` or `beta.example.com` + re := `^((www|beta)\.)?example\.com/` + if matched, _ := regexp.MatchString(re, req.URL.Host); matched { + return nil + } + return errors.New("Invalid redirect") +} From fd306ed79be81f5305fbba653f8e88e967dce552 Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Wed, 24 Apr 2024 12:19:11 +0100 Subject: [PATCH 23/39] Exclude constant names from sources to avoid duplicate results --- go/ql/src/Security/CWE-020/IncompleteHostnameRegexp.ql | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/go/ql/src/Security/CWE-020/IncompleteHostnameRegexp.ql b/go/ql/src/Security/CWE-020/IncompleteHostnameRegexp.ql index e114d924475..03018ee1c32 100644 --- a/go/ql/src/Security/CWE-020/IncompleteHostnameRegexp.ql +++ b/go/ql/src/Security/CWE-020/IncompleteHostnameRegexp.ql @@ -81,9 +81,12 @@ predicate regexpGuardsError(RegexpPattern regexp) { module IncompleteHostNameRegexpConfig implements DataFlow::ConfigSig { additional predicate isSourceString(DataFlow::Node source, string hostPart) { - exists(Expr e | - e = source.asExpr() and - isIncompleteHostNameRegexpPattern(e.getStringValue(), hostPart) + exists(Expr e | e = source.asExpr() | + isIncompleteHostNameRegexpPattern(e.getStringValue(), hostPart) and + // Exclude constant names to avoid duplicate results, because the string + // literals which they are initialised with are also considered as + // sources. + not e instanceof ConstantName ) } From 41409424795de9b79bc8ddbd4c64e73b50578059 Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Wed, 24 Apr 2024 12:20:18 +0100 Subject: [PATCH 24/39] Update tests --- .../IncompleteHostnameRegexp.expected | 18 +++++++++++---- .../CWE-020/IncompleteHostnameRegexp/main.go | 22 ++++++++++++++++++- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/go/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegexp/IncompleteHostnameRegexp.expected b/go/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegexp/IncompleteHostnameRegexp.expected index 4486b4e0962..c9ba782fd56 100644 --- a/go/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegexp/IncompleteHostnameRegexp.expected +++ b/go/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegexp/IncompleteHostnameRegexp.expected @@ -1,12 +1,22 @@ edges | IncompleteHostnameRegexp.go:11:8:11:36 | "^((www\|beta).)?example.com/" | IncompleteHostnameRegexp.go:12:38:12:39 | re | provenance | | +| main.go:49:21:49:45 | `https://www.example.com` | main.go:62:15:62:25 | sourceConst | provenance | | +| main.go:62:15:62:25 | sourceConst | main.go:65:15:65:23 | localVar3 | provenance | | nodes | IncompleteHostnameRegexp.go:11:8:11:36 | "^((www\|beta).)?example.com/" | semmle.label | "^((www\|beta).)?example.com/" | | IncompleteHostnameRegexp.go:12:38:12:39 | re | semmle.label | re | -| main.go:39:60:39:79 | "^test2.github.com$" | semmle.label | "^test2.github.com$" | -| main.go:44:15:44:39 | `https://www.example.com` | semmle.label | `https://www.example.com` | +| main.go:40:60:40:79 | "^test2.github.com$" | semmle.label | "^test2.github.com$" | +| main.go:45:15:45:39 | `https://www.example.com` | semmle.label | `https://www.example.com` | +| main.go:49:21:49:45 | `https://www.example.com` | semmle.label | `https://www.example.com` | +| main.go:56:15:56:34 | ...+... | semmle.label | ...+... | +| main.go:58:15:58:42 | ...+... | semmle.label | ...+... | +| main.go:62:15:62:25 | sourceConst | semmle.label | sourceConst | +| main.go:65:15:65:23 | localVar3 | semmle.label | localVar3 | subpaths #select | IncompleteHostnameRegexp.go:11:8:11:36 | "^((www\|beta).)?example.com/" | IncompleteHostnameRegexp.go:11:8:11:36 | "^((www\|beta).)?example.com/" | IncompleteHostnameRegexp.go:12:38:12:39 | re | This regular expression has an unescaped dot before ')?example.com', so it might match more hosts than expected when $@. | IncompleteHostnameRegexp.go:12:38:12:39 | re | the regular expression is used | -| main.go:39:60:39:79 | "^test2.github.com$" | main.go:39:60:39:79 | "^test2.github.com$" | main.go:39:60:39:79 | "^test2.github.com$" | This regular expression has an unescaped dot before 'github.com', so it might match more hosts than expected when $@. | main.go:39:60:39:79 | "^test2.github.com$" | the regular expression is used | -| main.go:44:15:44:39 | `https://www.example.com` | main.go:44:15:44:39 | `https://www.example.com` | main.go:44:15:44:39 | `https://www.example.com` | This regular expression has an unescaped dot before 'example.com', so it might match more hosts than expected when $@. | main.go:44:15:44:39 | `https://www.example.com` | the regular expression is used | +| main.go:40:60:40:79 | "^test2.github.com$" | main.go:40:60:40:79 | "^test2.github.com$" | main.go:40:60:40:79 | "^test2.github.com$" | This regular expression has an unescaped dot before 'github.com', so it might match more hosts than expected when $@. | main.go:40:60:40:79 | "^test2.github.com$" | the regular expression is used | +| main.go:45:15:45:39 | `https://www.example.com` | main.go:45:15:45:39 | `https://www.example.com` | main.go:45:15:45:39 | `https://www.example.com` | This regular expression has an unescaped dot before 'example.com', so it might match more hosts than expected when $@. | main.go:45:15:45:39 | `https://www.example.com` | the regular expression is used | +| main.go:49:21:49:45 | `https://www.example.com` | main.go:49:21:49:45 | `https://www.example.com` | main.go:65:15:65:23 | localVar3 | This regular expression has an unescaped dot before 'example.com', so it might match more hosts than expected when $@. | main.go:65:15:65:23 | localVar3 | the regular expression is used | +| main.go:56:15:56:34 | ...+... | main.go:56:15:56:34 | ...+... | main.go:56:15:56:34 | ...+... | This regular expression has an unescaped dot before 'example.com', so it might match more hosts than expected when $@. | main.go:56:15:56:34 | ...+... | the regular expression is used | +| main.go:58:15:58:42 | ...+... | main.go:58:15:58:42 | ...+... | main.go:58:15:58:42 | ...+... | This regular expression has an unescaped dot before 'example.com', so it might match more hosts than expected when $@. | main.go:58:15:58:42 | ...+... | the regular expression is used | diff --git a/go/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegexp/main.go b/go/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegexp/main.go index fb1e5b77b51..7eda0d7255a 100644 --- a/go/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegexp/main.go +++ b/go/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegexp/main.go @@ -3,10 +3,11 @@ package main import ( - "github.com/elazarl/goproxy" "net/http" "regexp" "time" + + "github.com/elazarl/goproxy" ) func Match(notARegex string) bool { @@ -44,3 +45,22 @@ func main() { regexp.Match(`https://www.example.com`, []byte("")) // NOT OK regexp.Match(`https://www\.example\.com`, []byte("")) // OK } + +const sourceConst = `https://www.example.com` +const firstHalfConst = `https://www.example.` + +func concatenateStrings() { + firstHalf := `https://www.example.` + regexp.Match(firstHalf+`com`, []byte("")) // MISSING: NOT OK + + regexp.Match(firstHalfConst+`com`, []byte("")) // NOT OK + + regexp.Match(`https://www.example.`+`com`, []byte("")) // NOT OK +} + +func avoidDuplicateResults() { + localVar1 := sourceConst + localVar2 := localVar1 + localVar3 := localVar2 + regexp.Match(localVar3, []byte("")) // NOT OK +} From c61177cf42d45784910d6999b625592f3f2fbb2d Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Wed, 24 Apr 2024 14:18:51 +0100 Subject: [PATCH 25/39] Add change note --- .../src/change-notes/2024-04-24-incomplete-hostname-regexp.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 go/ql/src/change-notes/2024-04-24-incomplete-hostname-regexp.md diff --git a/go/ql/src/change-notes/2024-04-24-incomplete-hostname-regexp.md b/go/ql/src/change-notes/2024-04-24-incomplete-hostname-regexp.md new file mode 100644 index 00000000000..3e7f0d593ec --- /dev/null +++ b/go/ql/src/change-notes/2024-04-24-incomplete-hostname-regexp.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* The query `go/incomplete-hostname-regexp` now recognizes more sources involving concatenation of string literals and also follows flow through string concatenation. This may lead to more alerts. From 9f5782b67b4d62b410d2a1002c44983b260de55c Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 24 Apr 2024 15:49:48 +0200 Subject: [PATCH 26/39] Bazel: introduce buildifier formatting This introduces tooling and enforcement for formatting bazel files. The tooling is provided as a bazel run target from [keith/buildifier-prebuilt](https://github.com/keith/buildifier-prebuilt). This is used in a [`pre-commit`](https://pre-commit.com/) hook for those having that installed. In turn this is used in a CI check. Relying on a `pre-commit` action gives us easy checking that buildifying did not change anything in the files and printing the diff, without having to hand-roll the check ourselves. This enforcement will make usage of gazelle easier, as gazelle itself might reformat files, even outside of `go`. Having them properly formatted will allow gazelle to leave them unchanged, without needing to configure awkward exclude directives. --- .github/workflows/buildifier.yml | 26 +++++++++++++++++++++ .pre-commit-config.yaml | 12 ++++++---- BUILD.bazel | 9 +++++++ MODULE.bazel | 2 ++ javascript/BUILD.bazel | 2 +- javascript/extractor/BUILD.bazel | 2 +- misc/codegen/BUILD.bazel | 2 -- misc/codegen/generators/BUILD.bazel | 2 -- python/extractor/tsg-python/tsp/BUILD.bazel | 2 +- swift/extractor/BUILD.bazel | 10 ++++---- swift/logging/BUILD.bazel | 2 +- 11 files changed, 53 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/buildifier.yml diff --git a/.github/workflows/buildifier.yml b/.github/workflows/buildifier.yml new file mode 100644 index 00000000000..d5cea415d25 --- /dev/null +++ b/.github/workflows/buildifier.yml @@ -0,0 +1,26 @@ +name: Check bazel formatting + +on: + pull_request: + paths: + - "**.bazel" + - "**.bzl" + branches: + - main + - "rc/*" + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507 + with: + extra_args: buildifier --all-files --show-diff-on-failure + - if: failure() + run: | + echo "In order to format all files, please run:" + echo " bazel run //:buildifier" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 383bc110383..7fddd1929d4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,13 +20,15 @@ repos: - id: autopep8 files: ^misc/codegen/.*\.py - - repo: https://github.com/warchant/pre-commit-buildifier - rev: 0.0.2 - hooks: - - id: buildifier - - repo: local hooks: + - id: buildifier + name: Format bazel files + files: \.(bazel|bzl) + language: system + entry: bazel run //:buildifier + pass_filenames: false + - id: codeql-format name: Fix QL file formatting files: \.qll?$ diff --git a/BUILD.bazel b/BUILD.bazel index e69de29bb2d..3ccdcda5f12 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -0,0 +1,9 @@ +load("@buildifier_prebuilt//:rules.bzl", "buildifier") + +buildifier( + name = "buildifier", + exclude_patterns = [ + "./.git/*", + ], + lint_mode = "fix", +) diff --git a/MODULE.bazel b/MODULE.bazel index 4e1fe0d9f7c..c834acab7ae 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -22,6 +22,8 @@ bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "absl") bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json") bazel_dep(name = "fmt", version = "10.0.0") +bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True) + pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") pip.parse( hub_name = "codegen_deps", diff --git a/javascript/BUILD.bazel b/javascript/BUILD.bazel index eacfa554a8e..18315a09813 100644 --- a/javascript/BUILD.bazel +++ b/javascript/BUILD.bazel @@ -1,5 +1,5 @@ -load("@semmle_code//:dist.bzl", "dist") load("@rules_pkg//pkg:mappings.bzl", "pkg_files") +load("@semmle_code//:dist.bzl", "dist") load("@semmle_code//buildutils-internal:zipmerge.bzl", "zipmerge") package(default_visibility = ["//visibility:public"]) diff --git a/javascript/extractor/BUILD.bazel b/javascript/extractor/BUILD.bazel index 6793287c22f..360dc9370f9 100644 --- a/javascript/extractor/BUILD.bazel +++ b/javascript/extractor/BUILD.bazel @@ -1,5 +1,5 @@ -load("@semmle_code//:common.bzl", "codeql_fat_jar", "codeql_java_project") load("@rules_pkg//pkg:mappings.bzl", "pkg_files") +load("@semmle_code//:common.bzl", "codeql_fat_jar", "codeql_java_project") java_library( name = "deps", diff --git a/misc/codegen/BUILD.bazel b/misc/codegen/BUILD.bazel index 102e52fb10b..52a5c001134 100644 --- a/misc/codegen/BUILD.bazel +++ b/misc/codegen/BUILD.bazel @@ -1,5 +1,3 @@ -load("@codegen_deps//:requirements.bzl", "requirement") - py_binary( name = "codegen", srcs = ["codegen.py"], diff --git a/misc/codegen/generators/BUILD.bazel b/misc/codegen/generators/BUILD.bazel index df328a7df7f..f731c42ce23 100644 --- a/misc/codegen/generators/BUILD.bazel +++ b/misc/codegen/generators/BUILD.bazel @@ -1,5 +1,3 @@ -load("@codegen_deps//:requirements.bzl", "requirement") - py_library( name = "generators", srcs = glob(["*.py"]), diff --git a/python/extractor/tsg-python/tsp/BUILD.bazel b/python/extractor/tsg-python/tsp/BUILD.bazel index e3389fce1cc..1f7ec006c65 100644 --- a/python/extractor/tsg-python/tsp/BUILD.bazel +++ b/python/extractor/tsg-python/tsp/BUILD.bazel @@ -1,6 +1,6 @@ +load("@py_deps//:defs.bzl", "aliases", "all_crate_deps") load("@rules_rust//cargo:defs.bzl", "cargo_build_script") load("@rules_rust//rust:defs.bzl", "rust_library") -load("@py_deps//:defs.bzl", "aliases", "all_crate_deps") package(default_visibility = ["//visibility:public"]) diff --git a/swift/extractor/BUILD.bazel b/swift/extractor/BUILD.bazel index 42c1105053c..3acdbf014e3 100644 --- a/swift/extractor/BUILD.bazel +++ b/swift/extractor/BUILD.bazel @@ -1,5 +1,5 @@ -load("//swift:rules.bzl", "swift_cc_binary") load("//misc/bazel:pkg_runfiles.bzl", "pkg_runfiles") +load("//swift:rules.bzl", "swift_cc_binary") swift_cc_binary( name = "extractor.real", @@ -7,6 +7,10 @@ swift_cc_binary( "*.h", "*.cpp", ]), + linkopts = select({ + "@platforms//os:macos": ["-headerpad_max_install_names"], + "//conditions:default": [], + }), visibility = ["//swift:__pkg__"], deps = [ "//swift/extractor/config", @@ -17,10 +21,6 @@ swift_cc_binary( "//swift/third_party/swift-llvm-support", "@absl//absl/strings", ], - linkopts = select({ - "@platforms//os:macos": ["-headerpad_max_install_names"], - "//conditions:default": [], - }), ) sh_binary( diff --git a/swift/logging/BUILD.bazel b/swift/logging/BUILD.bazel index cd2f3344166..1d6192b3c13 100644 --- a/swift/logging/BUILD.bazel +++ b/swift/logging/BUILD.bazel @@ -4,10 +4,10 @@ cc_library( hdrs = glob(["*.h"]), visibility = ["//visibility:public"], deps = [ + "//shared/cpp:extractor_shared", "@absl//absl/strings", "@binlog", "@fmt", "@json", - "//shared/cpp:extractor_shared", ], ) From 9af9873e04dedfae430d40eeeffc6e9125ade6e7 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 24 Apr 2024 16:20:54 +0200 Subject: [PATCH 27/39] CI: add names to steps --- .github/workflows/buildifier.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/buildifier.yml b/.github/workflows/buildifier.yml index d5cea415d25..f2a9dc88090 100644 --- a/.github/workflows/buildifier.yml +++ b/.github/workflows/buildifier.yml @@ -16,11 +16,15 @@ jobs: check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507 + - name: Checkout + uses: actions/checkout@v4 + - name: Check bazel formatting + uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507 with: extra_args: buildifier --all-files --show-diff-on-failure - - if: failure() + - name: Print rectifying command + if: failure() run: | - echo "In order to format all files, please run:" + echo "In order to format all bazel files, please run:" echo " bazel run //:buildifier" + exit 1 From 9def57250dfe303cacf80daefbe5e2dabb818613 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 24 Apr 2024 16:34:07 +0200 Subject: [PATCH 28/39] CI: make reporting better --- .github/workflows/buildifier.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/buildifier.yml b/.github/workflows/buildifier.yml index f2a9dc88090..3cdfb046c58 100644 --- a/.github/workflows/buildifier.yml +++ b/.github/workflows/buildifier.yml @@ -21,10 +21,11 @@ jobs: - name: Check bazel formatting uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507 with: - extra_args: buildifier --all-files --show-diff-on-failure - - name: Print rectifying command + extra_args: buildifier --all-files 2>&1 | tee check_output + - name: Report if: failure() run: | + sed -ne '/diff --git/,$ p' check_output echo "In order to format all bazel files, please run:" echo " bazel run //:buildifier" exit 1 From 196b6d7a1d1921b322b64db775c1d3d349ea0c7a Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 24 Apr 2024 16:43:20 +0200 Subject: [PATCH 29/39] CI: simplify reporting --- .github/workflows/buildifier.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/buildifier.yml b/.github/workflows/buildifier.yml index 3cdfb046c58..82444d0440a 100644 --- a/.github/workflows/buildifier.yml +++ b/.github/workflows/buildifier.yml @@ -21,11 +21,8 @@ jobs: - name: Check bazel formatting uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507 with: - extra_args: buildifier --all-files 2>&1 | tee check_output - - name: Report - if: failure() - run: | - sed -ne '/diff --git/,$ p' check_output - echo "In order to format all bazel files, please run:" - echo " bazel run //:buildifier" - exit 1 + extra_args: > + buildifier --all-files 2>&1 || + ( + echo -e "In order to format all bazel files, please run:\n bazel run //:buildifier"; exit 1 + ) From 95ec4e8d26dea482dd54e9d56038734e32dbabbc Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Wed, 24 Apr 2024 21:47:47 +0200 Subject: [PATCH 30/39] C++: Fix comment in IR test --- cpp/ql/test/library-tests/ir/ir/ir.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/ql/test/library-tests/ir/ir/ir.cpp b/cpp/ql/test/library-tests/ir/ir/ir.cpp index 17bfd2f9089..95b639aac6b 100644 --- a/cpp/ql/test/library-tests/ir/ir/ir.cpp +++ b/cpp/ql/test/library-tests/ir/ir/ir.cpp @@ -2432,7 +2432,7 @@ void initialization_with_temp_destructor() { } void param_with_destructor_by_value(ClassWithDestructor c) { - // The call to ~ClassWithDestructor::ClassWithDestructor() seems to be missing here. + // The call to ~ClassWithDestructor::ClassWithDestructor() happens on the side of the caller } void param_with_destructor_by_pointer(ClassWithDestructor* c) { From 15c1fd942558d2a5d5023dc5f17dfdd8dedd9757 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Thu, 25 Apr 2024 10:39:53 +0200 Subject: [PATCH 31/39] C#: Improve log messages --- .../DependencyManager.cs | 1 + .../DotNet.cs | 4 +-- .../DotNetCliInvoker.cs | 5 ++-- .../FileProvider.cs | 29 ++++++++++++------- .../NugetExeWrapper.cs | 24 ++++++--------- .../NugetPackageRestorer.cs | 4 +-- .../Extractor.cs | 9 ++++-- 7 files changed, 42 insertions(+), 34 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index d0e68defac1..d5450affd93 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -191,6 +191,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private HashSet AddFrameworkDlls(HashSet dllLocations) { + logger.LogInfo("Adding .NET Framework DLLs"); var frameworkLocations = new HashSet(); var frameworkReferences = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferences); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs index 95e8fe0675f..f1f1862049a 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs @@ -91,9 +91,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return dotnetCliInvoker.RunCommand(args); } - public IList GetListedRuntimes() => GetResultList("--list-runtimes", null, false); + public IList GetListedRuntimes() => GetResultList("--list-runtimes", null, true); - public IList GetListedSdks() => GetResultList("--list-sdks", null, false); + public IList GetListedSdks() => GetResultList("--list-sdks", null, true); private IList GetResultList(string args, string? workingDirectory = null, bool silent = true) { diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs index 1802521e28d..12678021227 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs @@ -19,6 +19,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { this.logger = logger; this.Exec = exec; + logger.LogInfo($"Using .NET CLI executable: '{Exec}'"); } private ProcessStartInfo MakeDotnetStartInfo(string args, string? workingDirectory) @@ -43,7 +44,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private bool RunCommandAux(string args, string? workingDirectory, out IList output, bool silent) { var dirLog = string.IsNullOrWhiteSpace(workingDirectory) ? "" : $" in {workingDirectory}"; - logger.LogInfo($"Running {Exec} {args}{dirLog}"); + logger.LogInfo($"Running '{Exec} {args}'{dirLog}"); var pi = MakeDotnetStartInfo(args, workingDirectory); var threadId = Environment.CurrentManagedThreadId; void onOut(string s) => logger.Log(silent ? Severity.Debug : Severity.Info, s, threadId); @@ -51,7 +52,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var exitCode = pi.ReadOutput(out output, onOut, onError); if (exitCode != 0) { - logger.LogError($"Command {Exec} {args}{dirLog} failed with exit code {exitCode}"); + logger.LogError($"Command '{Exec} {args}'{dirLog} failed with exit code {exitCode}"); return false; } return true; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileProvider.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileProvider.cs index f5ba20cd888..7b88a1fc1a2 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileProvider.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileProvider.cs @@ -21,6 +21,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private readonly Lazy dlls; private readonly Lazy nugetConfigs; private readonly Lazy globalJsons; + private readonly Lazy packagesConfigs; private readonly Lazy razorViews; private readonly Lazy resources; private readonly Lazy rootNugetConfig; @@ -32,31 +33,38 @@ namespace Semmle.Extraction.CSharp.DependencyFetching all = GetAllFiles(); allNonBinary = new Lazy(() => all.Where(f => !binaryFileExtensions.Contains(f.Extension.ToLowerInvariant())).ToArray()); - smallNonBinary = new Lazy(() => - { - var ret = SelectSmallFiles(allNonBinary.Value).SelectFileNames().ToArray(); - logger.LogInfo($"Found {ret.Length} small non-binary files in {SourceDir}."); - return ret; - }); + smallNonBinary = new Lazy(() => ReturnAndLogFiles("small non-binary", SelectSmallFiles(allNonBinary.Value).SelectFileNames().ToArray())); sources = new Lazy(() => SelectTextFileNamesByExtension("source", ".cs")); projects = new Lazy(() => SelectTextFileNamesByExtension("project", ".csproj")); solutions = new Lazy(() => SelectTextFileNamesByExtension("solution", ".sln")); dlls = new Lazy(() => SelectBinaryFileNamesByExtension("DLL", ".dll")); - nugetConfigs = new Lazy(() => allNonBinary.Value.SelectFileNamesByName("nuget.config").ToArray()); - globalJsons = new Lazy(() => allNonBinary.Value.SelectFileNamesByName("global.json").ToArray()); + nugetConfigs = new Lazy(() => SelectTextFileNamesByName("nuget.config")); + globalJsons = new Lazy(() => SelectTextFileNamesByName("global.json")); + packagesConfigs = new Lazy(() => SelectTextFileNamesByName("packages.config")); razorViews = new Lazy(() => SelectTextFileNamesByExtension("razor view", ".cshtml", ".razor")); resources = new Lazy(() => SelectTextFileNamesByExtension("resource", ".resx")); rootNugetConfig = new Lazy(() => all.SelectRootFiles(SourceDir).SelectFileNamesByName("nuget.config").FirstOrDefault()); } - private string[] SelectTextFileNamesByExtension(string filetype, params string[] extensions) + private string[] ReturnAndLogFiles(string filetype, IEnumerable files) { - var ret = allNonBinary.Value.SelectFileNamesByExtension(extensions).ToArray(); + var ret = files.ToArray(); logger.LogInfo($"Found {ret.Length} {filetype} files in {SourceDir}."); return ret; } + private string[] SelectTextFileNamesByExtension(string filetype, params string[] extensions) + => ReturnAndLogFiles(filetype, allNonBinary.Value.SelectFileNamesByExtension(extensions)); + + private string[] SelectTextFileNamesByName(string name) + { + var ret = allNonBinary.Value.SelectFileNamesByName(name).ToArray(); + var ending = ret.Length == 0 ? "." : $": {string.Join(", ", ret.OrderBy(s => s))}."; + logger.LogInfo($"Found {ret.Length} {name} files in {SourceDir}{ending}"); + return ret; + } + private string[] SelectBinaryFileNamesByExtension(string filetype, params string[] extensions) { var ret = all.SelectFileNamesByExtension(extensions).ToArray(); @@ -117,6 +125,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching public ICollection NugetConfigs => nugetConfigs.Value; public string? RootNugetConfig => rootNugetConfig.Value; public IEnumerable GlobalJsons => globalJsons.Value; + public ICollection PackagesConfigs => packagesConfigs.Value; public ICollection RazorViews => razorViews.Value; public ICollection Resources => resources.Value; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs index 01b4eb6b5f4..8537e4b5e0e 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs @@ -20,9 +20,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// /// The list of package files. /// - private readonly FileInfo[] packageFiles; + private readonly ICollection packageFiles; - public int PackageCount => packageFiles.Length; + public int PackageCount => packageFiles.Count; private readonly string? backupNugetConfig; private readonly string? nugetConfigPath; @@ -37,23 +37,21 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// /// Create the package manager for a specified source tree. /// - public NugetExeWrapper(string sourceDir, TemporaryDirectory packageDirectory, Util.Logging.ILogger logger) + public NugetExeWrapper(FileProvider fileProvider, TemporaryDirectory packageDirectory, Util.Logging.ILogger logger) { this.packageDirectory = packageDirectory; this.logger = logger; - packageFiles = new DirectoryInfo(sourceDir) - .EnumerateFiles("packages.config", SearchOption.AllDirectories) - .ToArray(); + packageFiles = fileProvider.PackagesConfigs; - if (packageFiles.Length > 0) + if (packageFiles.Count > 0) { - logger.LogInfo($"Found {packageFiles.Length} packages.config files, trying to use nuget.exe for package restore"); - nugetExe = ResolveNugetExe(sourceDir); + logger.LogInfo($"Found packages.config files, trying to use nuget.exe for package restore"); + nugetExe = ResolveNugetExe(fileProvider.SourceDir.FullName); if (HasNoPackageSource()) { // We only modify or add a top level nuget.config file - nugetConfigPath = Path.Combine(sourceDir, "nuget.config"); + nugetConfigPath = Path.Combine(fileProvider.SourceDir.FullName, "nuget.config"); try { if (File.Exists(nugetConfigPath)) @@ -86,10 +84,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } } - else - { - logger.LogInfo("Found no packages.config file"); - } } /// @@ -195,7 +189,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// public int InstallPackages() { - return packageFiles.Count(package => TryRestoreNugetPackage(package.FullName)); + return packageFiles.Count(package => TryRestoreNugetPackage(package)); } private bool HasNoPackageSource() diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index 735e4a676c6..baf23a0d0cc 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -105,7 +105,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching : [unresponsiveMissingPackageLocation]; } - using (var nuget = new NugetExeWrapper(fileProvider.SourceDir.FullName, legacyPackageDirectory, logger)) + using (var nuget = new NugetExeWrapper(fileProvider, legacyPackageDirectory, logger)) { var count = nuget.InstallPackages(); @@ -178,7 +178,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching logger.LogInfo($"No fallback Nuget feeds specified. Using default feed: {PublicNugetOrgFeed}"); } - logger.LogInfo($"Checking fallback Nuget feed reachability on feeds: {string.Join(", ", fallbackFeeds.OrderBy(f => f))}"); + logger.LogInfo($"Checking fallback Nuget feed reachability on feeds: {string.Join(", ", fallbackFeeds.OrderBy(f => f))}"); var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: true); var reachableFallbackFeeds = fallbackFeeds.Where(feed => IsFeedReachable(feed, initialTimeout, tryCount, allowExceptions: false)).ToList(); if (reachableFallbackFeeds.Count == 0) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs index fac8523989e..63d4ff0e83a 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs @@ -121,17 +121,20 @@ namespace Semmle.Extraction.CSharp.Standalone public void MissingType(string type) { - logger.Log(Severity.Debug, "Missing type {0}", type); + logger.LogDebug($"Missing type {type}"); } public void MissingNamespace(string @namespace) { - logger.Log(Severity.Info, "Missing namespace {0}", @namespace); + logger.LogInfo($"Missing namespace {@namespace}"); } public void MissingSummary(int missingTypes, int missingNamespaces) { - logger.Log(Severity.Info, "Failed to resolve {0} types in {1} namespaces", missingTypes, missingNamespaces); + if (missingTypes > 0 || missingNamespaces > 0) + { + logger.LogInfo($"Failed to resolve {missingTypes} types in {missingNamespaces} namespaces"); + } } } From 05819a52ef15a456af6fdc11daebe23a1badd84f Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Thu, 25 Apr 2024 13:20:21 +0200 Subject: [PATCH 32/39] C++: Print destructors for children of statements that are again statements --- cpp/ql/lib/semmle/code/cpp/PrintAST.qll | 26 +++ .../library-tests/ir/ir/PrintAST.expected | 108 +++++++++++++ .../library-tests/ir/ir/aliased_ir.expected | 153 ++++++++++++++++++ cpp/ql/test/library-tests/ir/ir/ir.cpp | 17 ++ .../test/library-tests/ir/ir/raw_ir.expected | 128 +++++++++++++++ 5 files changed, 432 insertions(+) diff --git a/cpp/ql/lib/semmle/code/cpp/PrintAST.qll b/cpp/ql/lib/semmle/code/cpp/PrintAST.qll index beabef322d2..b515a346bf3 100644 --- a/cpp/ql/lib/semmle/code/cpp/PrintAST.qll +++ b/cpp/ql/lib/semmle/code/cpp/PrintAST.qll @@ -463,6 +463,25 @@ class StmtNode extends AstNode { } } +/** + * A node representing a child of a `Stmt` that is itself a `Stmt`. + */ +class ChildStmtNode extends StmtNode { + Stmt childStmt; + + ChildStmtNode() { exists(Stmt parent | parent.getAChild() = childStmt and childStmt = ast) } + + override BaseAstNode getChildInternal(int childIndex) { + result = super.getChildInternal(childIndex) + or + exists(int destructorIndex | + result.getAst() = childStmt.getImplicitDestructorCall(destructorIndex) and + childIndex = + destructorIndex + max(int index | exists(childStmt.getChild(index)) or index = 0) + 1 + ) + } +} + /** * A node representing a `DeclStmt`. */ @@ -674,6 +693,13 @@ class FunctionNode extends FunctionOrGlobalOrNamespaceVariableNode { private string getChildAccessorWithoutConversions(Locatable parent, Element child) { shouldPrintDeclaration(getAnEnclosingDeclaration(parent)) and ( + exists(Stmt s, int i | s.getChild(i) = parent | + exists(int n | + s.getChild(i).(Stmt).getImplicitDestructorCall(n) = child and + result = "getImplicitDestructorCall(" + n + ")" + ) + ) + or exists(Stmt s | s = parent | namedStmtChildPredicates(s, child, result) or diff --git a/cpp/ql/test/library-tests/ir/ir/PrintAST.expected b/cpp/ql/test/library-tests/ir/ir/PrintAST.expected index 1f6a29b57ed..e0d2da046b2 100644 --- a/cpp/ql/test/library-tests/ir/ir/PrintAST.expected +++ b/cpp/ql/test/library-tests/ir/ir/PrintAST.expected @@ -22334,6 +22334,114 @@ ir.cpp: # 2480| Type = [Struct] B # 2480| ValueCategory = xvalue # 2481| getStmt(1): [ReturnStmt] return ... +# 2484| [TopLevelFunction] void destructor_without_block(bool) +# 2484| : +# 2484| getParameter(0): [Parameter] b +# 2484| Type = [BoolType] bool +# 2485| getEntryPoint(): [BlockStmt] { ... } +# 2486| getStmt(0): [IfStmt] if (...) ... +# 2486| getCondition(): [VariableAccess] b +# 2486| Type = [BoolType] bool +# 2486| ValueCategory = prvalue(load) +# 2487| getThen(): [DeclStmt] declaration +# 2487| getDeclarationEntry(0): [VariableDeclarationEntry] definition of c +# 2487| Type = [Class] ClassWithDestructor +# 2487| getVariable().getInitializer(): [Initializer] initializer for c +# 2487| getExpr(): [ConstructorCall] call to ClassWithDestructor +# 2487| Type = [VoidType] void +# 2487| ValueCategory = prvalue +#-----| getImplicitDestructorCall(0): [DestructorCall] call to ~ClassWithDestructor +#-----| Type = [VoidType] void +#-----| ValueCategory = prvalue +#-----| getQualifier(): [VariableAccess] c +#-----| Type = [Class] ClassWithDestructor +#-----| ValueCategory = lvalue +# 2489| getStmt(1): [IfStmt] if (...) ... +# 2489| getCondition(): [VariableAccess] b +# 2489| Type = [BoolType] bool +# 2489| ValueCategory = prvalue(load) +# 2490| getThen(): [DeclStmt] declaration +# 2490| getDeclarationEntry(0): [VariableDeclarationEntry] definition of d +# 2490| Type = [Class] ClassWithDestructor +# 2490| getVariable().getInitializer(): [Initializer] initializer for d +# 2490| getExpr(): [ConstructorCall] call to ClassWithDestructor +# 2490| Type = [VoidType] void +# 2490| ValueCategory = prvalue +#-----| getImplicitDestructorCall(0): [DestructorCall] call to ~ClassWithDestructor +#-----| Type = [VoidType] void +#-----| ValueCategory = prvalue +#-----| getQualifier(): [VariableAccess] d +#-----| Type = [Class] ClassWithDestructor +#-----| ValueCategory = lvalue +# 2492| getElse(): [DeclStmt] declaration +# 2492| getDeclarationEntry(0): [VariableDeclarationEntry] definition of e +# 2492| Type = [Class] ClassWithDestructor +# 2492| getVariable().getInitializer(): [Initializer] initializer for e +# 2492| getExpr(): [ConstructorCall] call to ClassWithDestructor +# 2492| Type = [VoidType] void +# 2492| ValueCategory = prvalue +#-----| getImplicitDestructorCall(0): [DestructorCall] call to ~ClassWithDestructor +#-----| Type = [VoidType] void +#-----| ValueCategory = prvalue +#-----| getQualifier(): [VariableAccess] e +#-----| Type = [Class] ClassWithDestructor +#-----| ValueCategory = lvalue +# 2494| getStmt(2): [WhileStmt] while (...) ... +# 2494| getCondition(): [VariableAccess] b +# 2494| Type = [BoolType] bool +# 2494| ValueCategory = prvalue(load) +# 2495| getStmt(): [DeclStmt] declaration +# 2495| getDeclarationEntry(0): [VariableDeclarationEntry] definition of f +# 2495| Type = [Class] ClassWithDestructor +# 2495| getVariable().getInitializer(): [Initializer] initializer for f +# 2495| getExpr(): [ConstructorCall] call to ClassWithDestructor +# 2495| Type = [VoidType] void +# 2495| ValueCategory = prvalue +#-----| getImplicitDestructorCall(0): [DestructorCall] call to ~ClassWithDestructor +#-----| Type = [VoidType] void +#-----| ValueCategory = prvalue +#-----| getQualifier(): [VariableAccess] f +#-----| Type = [Class] ClassWithDestructor +#-----| ValueCategory = lvalue +# 2497| getStmt(3): [ForStmt] for(...;...;...) ... +# 2497| getInitialization(): [DeclStmt] declaration +# 2497| getDeclarationEntry(0): [VariableDeclarationEntry] definition of i +# 2497| Type = [IntType] int +# 2497| getVariable().getInitializer(): [Initializer] initializer for i +# 2497| getExpr(): [Literal] 0 +# 2497| Type = [IntType] int +# 2497| Value = [Literal] 0 +# 2497| ValueCategory = prvalue +# 2497| getCondition(): [LTExpr] ... < ... +# 2497| Type = [BoolType] bool +# 2497| ValueCategory = prvalue +# 2497| getLesserOperand(): [VariableAccess] i +# 2497| Type = [IntType] int +# 2497| ValueCategory = prvalue(load) +# 2497| getGreaterOperand(): [Literal] 42 +# 2497| Type = [IntType] int +# 2497| Value = [Literal] 42 +# 2497| ValueCategory = prvalue +# 2497| getUpdate(): [PrefixIncrExpr] ++ ... +# 2497| Type = [IntType] int +# 2497| ValueCategory = lvalue +# 2497| getOperand(): [VariableAccess] i +# 2497| Type = [IntType] int +# 2497| ValueCategory = lvalue +# 2498| getStmt(): [DeclStmt] declaration +# 2498| getDeclarationEntry(0): [VariableDeclarationEntry] definition of g +# 2498| Type = [Class] ClassWithDestructor +# 2498| getVariable().getInitializer(): [Initializer] initializer for g +# 2498| getExpr(): [ConstructorCall] call to ClassWithDestructor +# 2498| Type = [VoidType] void +# 2498| ValueCategory = prvalue +#-----| getImplicitDestructorCall(0): [DestructorCall] call to ~ClassWithDestructor +#-----| Type = [VoidType] void +#-----| ValueCategory = prvalue +#-----| getQualifier(): [VariableAccess] g +#-----| Type = [Class] ClassWithDestructor +#-----| ValueCategory = lvalue +# 2499| getStmt(4): [ReturnStmt] return ... perf-regression.cpp: # 4| [CopyAssignmentOperator] Big& Big::operator=(Big const&) # 4| : diff --git a/cpp/ql/test/library-tests/ir/ir/aliased_ir.expected b/cpp/ql/test/library-tests/ir/ir/aliased_ir.expected index 51e12b7a6e5..3d3a56e790e 100644 --- a/cpp/ql/test/library-tests/ir/ir/aliased_ir.expected +++ b/cpp/ql/test/library-tests/ir/ir/aliased_ir.expected @@ -17805,6 +17805,159 @@ ir.cpp: # 2478| v2478_6(void) = AliasedUse : ~m2480_20 # 2478| v2478_7(void) = ExitFunction : +# 2484| void destructor_without_block(bool) +# 2484| Block 0 +# 2484| v2484_1(void) = EnterFunction : +# 2484| m2484_2(unknown) = AliasedDefinition : +# 2484| m2484_3(unknown) = InitializeNonLocal : +# 2484| m2484_4(unknown) = Chi : total:m2484_2, partial:m2484_3 +# 2484| r2484_5(glval) = VariableAddress[b] : +# 2484| m2484_6(bool) = InitializeParameter[b] : &:r2484_5 +# 2486| r2486_1(glval) = VariableAddress[b] : +# 2486| r2486_2(bool) = Load[b] : &:r2486_1, m2484_6 +# 2486| v2486_3(void) = ConditionalBranch : r2486_2 +#-----| False -> Block 2 +#-----| True -> Block 1 + +# 2487| Block 1 +# 2487| r2487_1(glval) = VariableAddress[c] : +# 2487| m2487_2(ClassWithDestructor) = Uninitialized[c] : &:r2487_1 +# 2487| r2487_3(glval) = FunctionAddress[ClassWithDestructor] : +# 2487| v2487_4(void) = Call[ClassWithDestructor] : func:r2487_3, this:r2487_1 +# 2487| m2487_5(unknown) = ^CallSideEffect : ~m2484_4 +# 2487| m2487_6(unknown) = Chi : total:m2484_4, partial:m2487_5 +# 2487| m2487_7(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2487_1 +# 2487| m2487_8(ClassWithDestructor) = Chi : total:m2487_2, partial:m2487_7 +#-----| r0_1(glval) = VariableAddress[c] : +#-----| r0_2(glval) = FunctionAddress[~ClassWithDestructor] : +#-----| v0_3(void) = Call[~ClassWithDestructor] : func:r0_2, this:r0_1 +#-----| m0_4(unknown) = ^CallSideEffect : ~m2487_6 +#-----| m0_5(unknown) = Chi : total:m2487_6, partial:m0_4 +#-----| v0_6(void) = ^IndirectReadSideEffect[-1] : &:r0_1, m2487_8 +#-----| m0_7(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r0_1 +#-----| m0_8(ClassWithDestructor) = Chi : total:m2487_8, partial:m0_7 +#-----| Goto -> Block 2 + +# 2489| Block 2 +# 2489| m2489_1(unknown) = Phi : from 0:~m2484_4, from 1:~m0_5 +# 2489| r2489_2(glval) = VariableAddress[b] : +# 2489| r2489_3(bool) = Load[b] : &:r2489_2, m2484_6 +# 2489| v2489_4(void) = ConditionalBranch : r2489_3 +#-----| False -> Block 4 +#-----| True -> Block 3 + +# 2490| Block 3 +# 2490| r2490_1(glval) = VariableAddress[d] : +# 2490| m2490_2(ClassWithDestructor) = Uninitialized[d] : &:r2490_1 +# 2490| r2490_3(glval) = FunctionAddress[ClassWithDestructor] : +# 2490| v2490_4(void) = Call[ClassWithDestructor] : func:r2490_3, this:r2490_1 +# 2490| m2490_5(unknown) = ^CallSideEffect : ~m2489_1 +# 2490| m2490_6(unknown) = Chi : total:m2489_1, partial:m2490_5 +# 2490| m2490_7(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2490_1 +# 2490| m2490_8(ClassWithDestructor) = Chi : total:m2490_2, partial:m2490_7 +#-----| r0_9(glval) = VariableAddress[d] : +#-----| r0_10(glval) = FunctionAddress[~ClassWithDestructor] : +#-----| v0_11(void) = Call[~ClassWithDestructor] : func:r0_10, this:r0_9 +#-----| m0_12(unknown) = ^CallSideEffect : ~m2490_6 +#-----| m0_13(unknown) = Chi : total:m2490_6, partial:m0_12 +#-----| v0_14(void) = ^IndirectReadSideEffect[-1] : &:r0_9, m2490_8 +#-----| m0_15(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r0_9 +#-----| m0_16(ClassWithDestructor) = Chi : total:m2490_8, partial:m0_15 +#-----| Goto -> Block 5 + +# 2492| Block 4 +# 2492| r2492_1(glval) = VariableAddress[e] : +# 2492| m2492_2(ClassWithDestructor) = Uninitialized[e] : &:r2492_1 +# 2492| r2492_3(glval) = FunctionAddress[ClassWithDestructor] : +# 2492| v2492_4(void) = Call[ClassWithDestructor] : func:r2492_3, this:r2492_1 +# 2492| m2492_5(unknown) = ^CallSideEffect : ~m2489_1 +# 2492| m2492_6(unknown) = Chi : total:m2489_1, partial:m2492_5 +# 2492| m2492_7(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2492_1 +# 2492| m2492_8(ClassWithDestructor) = Chi : total:m2492_2, partial:m2492_7 +#-----| r0_17(glval) = VariableAddress[e] : +#-----| r0_18(glval) = FunctionAddress[~ClassWithDestructor] : +#-----| v0_19(void) = Call[~ClassWithDestructor] : func:r0_18, this:r0_17 +#-----| m0_20(unknown) = ^CallSideEffect : ~m2492_6 +#-----| m0_21(unknown) = Chi : total:m2492_6, partial:m0_20 +#-----| v0_22(void) = ^IndirectReadSideEffect[-1] : &:r0_17, m2492_8 +#-----| m0_23(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r0_17 +#-----| m0_24(ClassWithDestructor) = Chi : total:m2492_8, partial:m0_23 +#-----| Goto -> Block 5 + +# 2494| Block 5 +# 2494| m2494_1(unknown) = Phi : from 3:~m0_13, from 4:~m0_21, from 6:~m0_29 +# 2494| r2494_2(glval) = VariableAddress[b] : +# 2494| r2494_3(bool) = Load[b] : &:r2494_2, m2484_6 +# 2494| v2494_4(void) = ConditionalBranch : r2494_3 +#-----| False -> Block 7 +#-----| True -> Block 6 + +# 2495| Block 6 +# 2495| r2495_1(glval) = VariableAddress[f] : +# 2495| m2495_2(ClassWithDestructor) = Uninitialized[f] : &:r2495_1 +# 2495| r2495_3(glval) = FunctionAddress[ClassWithDestructor] : +# 2495| v2495_4(void) = Call[ClassWithDestructor] : func:r2495_3, this:r2495_1 +# 2495| m2495_5(unknown) = ^CallSideEffect : ~m2494_1 +# 2495| m2495_6(unknown) = Chi : total:m2494_1, partial:m2495_5 +# 2495| m2495_7(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2495_1 +# 2495| m2495_8(ClassWithDestructor) = Chi : total:m2495_2, partial:m2495_7 +#-----| r0_25(glval) = VariableAddress[f] : +#-----| r0_26(glval) = FunctionAddress[~ClassWithDestructor] : +#-----| v0_27(void) = Call[~ClassWithDestructor] : func:r0_26, this:r0_25 +#-----| m0_28(unknown) = ^CallSideEffect : ~m2495_6 +#-----| m0_29(unknown) = Chi : total:m2495_6, partial:m0_28 +#-----| v0_30(void) = ^IndirectReadSideEffect[-1] : &:r0_25, m2495_8 +#-----| m0_31(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r0_25 +#-----| m0_32(ClassWithDestructor) = Chi : total:m2495_8, partial:m0_31 +#-----| Goto (back edge) -> Block 5 + +# 2497| Block 7 +# 2497| r2497_1(glval) = VariableAddress[i] : +# 2497| r2497_2(int) = Constant[0] : +# 2497| m2497_3(int) = Store[i] : &:r2497_1, r2497_2 +#-----| Goto -> Block 8 + +# 2497| Block 8 +# 2497| m2497_4(unknown) = Phi : from 7:~m2494_1, from 9:~m0_37 +# 2497| m2497_5(int) = Phi : from 7:m2497_3, from 9:m2497_15 +# 2497| r2497_6(glval) = VariableAddress[i] : +# 2497| r2497_7(int) = Load[i] : &:r2497_6, m2497_5 +# 2497| r2497_8(int) = Constant[42] : +# 2497| r2497_9(bool) = CompareLT : r2497_7, r2497_8 +# 2497| v2497_10(void) = ConditionalBranch : r2497_9 +#-----| False -> Block 10 +#-----| True -> Block 9 + +# 2498| Block 9 +# 2498| r2498_1(glval) = VariableAddress[g] : +# 2498| m2498_2(ClassWithDestructor) = Uninitialized[g] : &:r2498_1 +# 2498| r2498_3(glval) = FunctionAddress[ClassWithDestructor] : +# 2498| v2498_4(void) = Call[ClassWithDestructor] : func:r2498_3, this:r2498_1 +# 2498| m2498_5(unknown) = ^CallSideEffect : ~m2497_4 +# 2498| m2498_6(unknown) = Chi : total:m2497_4, partial:m2498_5 +# 2498| m2498_7(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2498_1 +# 2498| m2498_8(ClassWithDestructor) = Chi : total:m2498_2, partial:m2498_7 +#-----| r0_33(glval) = VariableAddress[g] : +#-----| r0_34(glval) = FunctionAddress[~ClassWithDestructor] : +#-----| v0_35(void) = Call[~ClassWithDestructor] : func:r0_34, this:r0_33 +#-----| m0_36(unknown) = ^CallSideEffect : ~m2498_6 +#-----| m0_37(unknown) = Chi : total:m2498_6, partial:m0_36 +#-----| v0_38(void) = ^IndirectReadSideEffect[-1] : &:r0_33, m2498_8 +#-----| m0_39(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r0_33 +#-----| m0_40(ClassWithDestructor) = Chi : total:m2498_8, partial:m0_39 +# 2497| r2497_11(glval) = VariableAddress[i] : +# 2497| r2497_12(int) = Load[i] : &:r2497_11, m2497_5 +# 2497| r2497_13(int) = Constant[1] : +# 2497| r2497_14(int) = Add : r2497_12, r2497_13 +# 2497| m2497_15(int) = Store[i] : &:r2497_11, r2497_14 +#-----| Goto (back edge) -> Block 8 + +# 2499| Block 10 +# 2499| v2499_1(void) = NoOp : +# 2484| v2484_7(void) = ReturnVoid : +# 2484| v2484_8(void) = AliasedUse : ~m2497_4 +# 2484| v2484_9(void) = ExitFunction : + perf-regression.cpp: # 6| void Big::Big() # 6| Block 0 diff --git a/cpp/ql/test/library-tests/ir/ir/ir.cpp b/cpp/ql/test/library-tests/ir/ir/ir.cpp index 95b639aac6b..c8840fd0b56 100644 --- a/cpp/ql/test/library-tests/ir/ir/ir.cpp +++ b/cpp/ql/test/library-tests/ir/ir/ir.cpp @@ -2481,4 +2481,21 @@ namespace rvalue_conversion_with_destructor { } } +void destructor_without_block(bool b) +{ + if (b) + ClassWithDestructor c; + + if (b) + ClassWithDestructor d; + else + ClassWithDestructor e; + + while (b) + ClassWithDestructor f; + + for(int i = 0; i < 42; ++i) + ClassWithDestructor g; +} + // semmle-extractor-options: -std=c++20 --clang diff --git a/cpp/ql/test/library-tests/ir/ir/raw_ir.expected b/cpp/ql/test/library-tests/ir/ir/raw_ir.expected index 7c0a525d0ca..6ff42a28cb3 100644 --- a/cpp/ql/test/library-tests/ir/ir/raw_ir.expected +++ b/cpp/ql/test/library-tests/ir/ir/raw_ir.expected @@ -16244,6 +16244,134 @@ ir.cpp: # 2478| v2478_5(void) = AliasedUse : ~m? # 2478| v2478_6(void) = ExitFunction : +# 2484| void destructor_without_block(bool) +# 2484| Block 0 +# 2484| v2484_1(void) = EnterFunction : +# 2484| mu2484_2(unknown) = AliasedDefinition : +# 2484| mu2484_3(unknown) = InitializeNonLocal : +# 2484| r2484_4(glval) = VariableAddress[b] : +# 2484| mu2484_5(bool) = InitializeParameter[b] : &:r2484_4 +# 2486| r2486_1(glval) = VariableAddress[b] : +# 2486| r2486_2(bool) = Load[b] : &:r2486_1, ~m? +# 2486| v2486_3(void) = ConditionalBranch : r2486_2 +#-----| False -> Block 2 +#-----| True -> Block 1 + +# 2487| Block 1 +# 2487| r2487_1(glval) = VariableAddress[c] : +# 2487| mu2487_2(ClassWithDestructor) = Uninitialized[c] : &:r2487_1 +# 2487| r2487_3(glval) = FunctionAddress[ClassWithDestructor] : +# 2487| v2487_4(void) = Call[ClassWithDestructor] : func:r2487_3, this:r2487_1 +# 2487| mu2487_5(unknown) = ^CallSideEffect : ~m? +# 2487| mu2487_6(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2487_1 +#-----| r0_1(glval) = VariableAddress[c] : +#-----| r0_2(glval) = FunctionAddress[~ClassWithDestructor] : +#-----| v0_3(void) = Call[~ClassWithDestructor] : func:r0_2, this:r0_1 +#-----| mu0_4(unknown) = ^CallSideEffect : ~m? +#-----| v0_5(void) = ^IndirectReadSideEffect[-1] : &:r0_1, ~m? +#-----| mu0_6(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r0_1 +#-----| Goto -> Block 2 + +# 2489| Block 2 +# 2489| r2489_1(glval) = VariableAddress[b] : +# 2489| r2489_2(bool) = Load[b] : &:r2489_1, ~m? +# 2489| v2489_3(void) = ConditionalBranch : r2489_2 +#-----| False -> Block 4 +#-----| True -> Block 3 + +# 2490| Block 3 +# 2490| r2490_1(glval) = VariableAddress[d] : +# 2490| mu2490_2(ClassWithDestructor) = Uninitialized[d] : &:r2490_1 +# 2490| r2490_3(glval) = FunctionAddress[ClassWithDestructor] : +# 2490| v2490_4(void) = Call[ClassWithDestructor] : func:r2490_3, this:r2490_1 +# 2490| mu2490_5(unknown) = ^CallSideEffect : ~m? +# 2490| mu2490_6(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2490_1 +#-----| r0_7(glval) = VariableAddress[d] : +#-----| r0_8(glval) = FunctionAddress[~ClassWithDestructor] : +#-----| v0_9(void) = Call[~ClassWithDestructor] : func:r0_8, this:r0_7 +#-----| mu0_10(unknown) = ^CallSideEffect : ~m? +#-----| v0_11(void) = ^IndirectReadSideEffect[-1] : &:r0_7, ~m? +#-----| mu0_12(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r0_7 +#-----| Goto -> Block 5 + +# 2492| Block 4 +# 2492| r2492_1(glval) = VariableAddress[e] : +# 2492| mu2492_2(ClassWithDestructor) = Uninitialized[e] : &:r2492_1 +# 2492| r2492_3(glval) = FunctionAddress[ClassWithDestructor] : +# 2492| v2492_4(void) = Call[ClassWithDestructor] : func:r2492_3, this:r2492_1 +# 2492| mu2492_5(unknown) = ^CallSideEffect : ~m? +# 2492| mu2492_6(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2492_1 +#-----| r0_13(glval) = VariableAddress[e] : +#-----| r0_14(glval) = FunctionAddress[~ClassWithDestructor] : +#-----| v0_15(void) = Call[~ClassWithDestructor] : func:r0_14, this:r0_13 +#-----| mu0_16(unknown) = ^CallSideEffect : ~m? +#-----| v0_17(void) = ^IndirectReadSideEffect[-1] : &:r0_13, ~m? +#-----| mu0_18(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r0_13 +#-----| Goto -> Block 5 + +# 2494| Block 5 +# 2494| r2494_1(glval) = VariableAddress[b] : +# 2494| r2494_2(bool) = Load[b] : &:r2494_1, ~m? +# 2494| v2494_3(void) = ConditionalBranch : r2494_2 +#-----| False -> Block 7 +#-----| True -> Block 6 + +# 2495| Block 6 +# 2495| r2495_1(glval) = VariableAddress[f] : +# 2495| mu2495_2(ClassWithDestructor) = Uninitialized[f] : &:r2495_1 +# 2495| r2495_3(glval) = FunctionAddress[ClassWithDestructor] : +# 2495| v2495_4(void) = Call[ClassWithDestructor] : func:r2495_3, this:r2495_1 +# 2495| mu2495_5(unknown) = ^CallSideEffect : ~m? +# 2495| mu2495_6(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2495_1 +#-----| r0_19(glval) = VariableAddress[f] : +#-----| r0_20(glval) = FunctionAddress[~ClassWithDestructor] : +#-----| v0_21(void) = Call[~ClassWithDestructor] : func:r0_20, this:r0_19 +#-----| mu0_22(unknown) = ^CallSideEffect : ~m? +#-----| v0_23(void) = ^IndirectReadSideEffect[-1] : &:r0_19, ~m? +#-----| mu0_24(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r0_19 +#-----| Goto (back edge) -> Block 5 + +# 2497| Block 7 +# 2497| r2497_1(glval) = VariableAddress[i] : +# 2497| r2497_2(int) = Constant[0] : +# 2497| mu2497_3(int) = Store[i] : &:r2497_1, r2497_2 +#-----| Goto -> Block 8 + +# 2497| Block 8 +# 2497| r2497_4(glval) = VariableAddress[i] : +# 2497| r2497_5(int) = Load[i] : &:r2497_4, ~m? +# 2497| r2497_6(int) = Constant[42] : +# 2497| r2497_7(bool) = CompareLT : r2497_5, r2497_6 +# 2497| v2497_8(void) = ConditionalBranch : r2497_7 +#-----| False -> Block 10 +#-----| True -> Block 9 + +# 2498| Block 9 +# 2498| r2498_1(glval) = VariableAddress[g] : +# 2498| mu2498_2(ClassWithDestructor) = Uninitialized[g] : &:r2498_1 +# 2498| r2498_3(glval) = FunctionAddress[ClassWithDestructor] : +# 2498| v2498_4(void) = Call[ClassWithDestructor] : func:r2498_3, this:r2498_1 +# 2498| mu2498_5(unknown) = ^CallSideEffect : ~m? +# 2498| mu2498_6(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r2498_1 +#-----| r0_25(glval) = VariableAddress[g] : +#-----| r0_26(glval) = FunctionAddress[~ClassWithDestructor] : +#-----| v0_27(void) = Call[~ClassWithDestructor] : func:r0_26, this:r0_25 +#-----| mu0_28(unknown) = ^CallSideEffect : ~m? +#-----| v0_29(void) = ^IndirectReadSideEffect[-1] : &:r0_25, ~m? +#-----| mu0_30(ClassWithDestructor) = ^IndirectMayWriteSideEffect[-1] : &:r0_25 +# 2497| r2497_9(glval) = VariableAddress[i] : +# 2497| r2497_10(int) = Load[i] : &:r2497_9, ~m? +# 2497| r2497_11(int) = Constant[1] : +# 2497| r2497_12(int) = Add : r2497_10, r2497_11 +# 2497| mu2497_13(int) = Store[i] : &:r2497_9, r2497_12 +#-----| Goto (back edge) -> Block 8 + +# 2499| Block 10 +# 2499| v2499_1(void) = NoOp : +# 2484| v2484_6(void) = ReturnVoid : +# 2484| v2484_7(void) = AliasedUse : ~m? +# 2484| v2484_8(void) = ExitFunction : + perf-regression.cpp: # 6| void Big::Big() # 6| Block 0 From d0c9e3f7ad64be3606267552e8bc9f49e721b4fe Mon Sep 17 00:00:00 2001 From: Asger F Date: Thu, 25 Apr 2024 13:33:17 +0200 Subject: [PATCH 33/39] JS: Expose InternalModuleNaming --- .../ql/lib/semmle/javascript/endpoints/EndpointNaming.qll | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/javascript/ql/lib/semmle/javascript/endpoints/EndpointNaming.qll b/javascript/ql/lib/semmle/javascript/endpoints/EndpointNaming.qll index 9e514d4c9f4..8fcb83729c4 100644 --- a/javascript/ql/lib/semmle/javascript/endpoints/EndpointNaming.qll +++ b/javascript/ql/lib/semmle/javascript/endpoints/EndpointNaming.qll @@ -432,7 +432,7 @@ string renderName(string package, string name) { result = join("(" + package + " * These names are not necessarily part of a package's public API, and so we only used them * as a fallback when a publicly-accessible access path cannot be found. */ -private module InternalModuleNaming { +module InternalModuleNaming { /** Gets the path to `folder` relative to its enclosing non-private `package.json` file. */ private string getPackageRelativePathFromFolder(Folder folder) { exists(PackageJson json | @@ -446,7 +446,10 @@ private module InternalModuleNaming { getPackageRelativePathFromFolder(folder.getParentContainer()) + "/" + folder.getBaseName() } - private string getPackageRelativePath(Module mod) { + /** + * Gets the path to `mod` relative to its enclosing package, including the package name. + */ + string getPackageRelativePath(Module mod) { exists(PackageJson json, string relativePath | not json.isPrivate() and json.getExportedModule(relativePath) = mod and From dd9183c3457d60cc3bece3f9cbafac3c416ce040 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Thu, 25 Apr 2024 14:38:26 +0200 Subject: [PATCH 34/39] Code quality improvements --- .../Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs index f1f1862049a..642403271ff 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs @@ -91,9 +91,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return dotnetCliInvoker.RunCommand(args); } - public IList GetListedRuntimes() => GetResultList("--list-runtimes", null, true); + public IList GetListedRuntimes() => GetResultList("--list-runtimes"); - public IList GetListedSdks() => GetResultList("--list-sdks", null, true); + public IList GetListedSdks() => GetResultList("--list-sdks"); private IList GetResultList(string args, string? workingDirectory = null, bool silent = true) { From 15a6308c7232c32df6e39c279180a8c27ef553e8 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Thu, 25 Apr 2024 14:04:00 +0100 Subject: [PATCH 35/39] Go: Refactor condition for `EmitInvalidToolchainVersion` into separate function --- go/extractor/project/project.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/go/extractor/project/project.go b/go/extractor/project/project.go index 4f6f6542f75..1d915282b95 100644 --- a/go/extractor/project/project.go +++ b/go/extractor/project/project.go @@ -179,6 +179,13 @@ func findGoModFiles(root string) []string { // A regular expression for the Go toolchain version syntax. var toolchainVersionRe *regexp.Regexp = regexp.MustCompile(`(?m)^([0-9]+\.[0-9]+\.[0-9]+)$`) +// Returns true if the `go.mod` file specifies a Go language version, that version is `1.21` or greater, and +// there is no `toolchain` directive, and the Go language version is not a valid toolchain version. +func hasInvalidToolchainVersion(modFile *modfile.File) bool { + return modFile.Toolchain == nil && modFile.Go != nil && + !toolchainVersionRe.Match([]byte(modFile.Go.Version)) && semver.Compare("v"+modFile.Go.Version, "v1.21.0") >= 0 +} + // Given a list of `go.mod` file paths, try to parse them all. The resulting array of `GoModule` objects // will be the same length as the input array and the objects will contain at least the `go.mod` path. // If parsing the corresponding file is successful, then the parsed contents will also be available. @@ -209,8 +216,7 @@ func LoadGoModules(emitDiagnostics bool, goModFilePaths []string) []*GoModule { // there is no `toolchain` directive, check that it is a valid Go toolchain version. Otherwise, // `go` commands which try to download the right version of the Go toolchain will fail. We detect // this situation and emit a diagnostic. - if modFile.Toolchain == nil && modFile.Go != nil && - !toolchainVersionRe.Match([]byte(modFile.Go.Version)) && semver.Compare("v"+modFile.Go.Version, "v1.21.0") >= 0 { + if hasInvalidToolchainVersion(modFile) { diagnostics.EmitInvalidToolchainVersion(goModFilePath, modFile.Go.Version) } } From f33d7ee80d8ae835e0244666ef87e41884d1fcc5 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Thu, 25 Apr 2024 14:09:47 +0100 Subject: [PATCH 36/39] Go: Add unit tests for `hasInvalidToolchainVersion` --- go/extractor/project/project_test.go | 37 ++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/go/extractor/project/project_test.go b/go/extractor/project/project_test.go index f2de420773f..b7485960b5f 100644 --- a/go/extractor/project/project_test.go +++ b/go/extractor/project/project_test.go @@ -3,6 +3,8 @@ package project import ( "path/filepath" "testing" + + "golang.org/x/mod/modfile" ) func testStartsWithAnyOf(t *testing.T, path string, prefix string, expectation bool) { @@ -25,3 +27,38 @@ func TestStartsWithAnyOf(t *testing.T) { testStartsWithAnyOf(t, filepath.Join("foo", "bar"), "bar", false) testStartsWithAnyOf(t, filepath.Join("foo", "bar"), filepath.Join("foo", "baz"), false) } + +func testHasInvalidToolchainVersion(t *testing.T, contents string) bool { + modFile, err := modfile.Parse("test.go", []byte(contents), nil) + + if err != nil { + t.Errorf("Unable to parse %s: %s.\n", contents, err.Error()) + } + + return hasInvalidToolchainVersion(modFile) +} + +func TestHasInvalidToolchainVersion(t *testing.T) { + invalid := []string{ + "go 1.21\n", + "go 1.22\n", + } + + for _, v := range invalid { + if !testHasInvalidToolchainVersion(t, v) { + t.Errorf("Expected testHasInvalidToolchainVersion(\"%s\") to be true, but got false", v) + } + } + + valid := []string{ + "go 1.20\n", + "go 1.21.1\n", + "go 1.22\n\ntoolchain go1.22.0\n", + } + + for _, v := range valid { + if testHasInvalidToolchainVersion(t, v) { + t.Errorf("Expected testHasInvalidToolchainVersion(\"%s\") to be false, but got true", v) + } + } +} From b8cfff6d199a616785d5bcc97690046db91dd141 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Thu, 25 Apr 2024 14:10:26 +0100 Subject: [PATCH 37/39] Go: Use `Parse` instead of `ParseLax`, since we need `toolchain` directives --- go/extractor/project/project.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/extractor/project/project.go b/go/extractor/project/project.go index 1d915282b95..215d7c6bc4f 100644 --- a/go/extractor/project/project.go +++ b/go/extractor/project/project.go @@ -203,7 +203,7 @@ func LoadGoModules(emitDiagnostics bool, goModFilePaths []string) []*GoModule { continue } - modFile, err := modfile.ParseLax(goModFilePath, modFileSrc, nil) + modFile, err := modfile.Parse(goModFilePath, modFileSrc, nil) if err != nil { log.Printf("Unable to parse %s: %s.\n", goModFilePath, err.Error()) From 14d04903dc9f858e9d58f8bd0b6152218f0143f0 Mon Sep 17 00:00:00 2001 From: Florin Coada Date: Thu, 25 Apr 2024 14:40:44 +0100 Subject: [PATCH 38/39] Update codeql-changelog index.rst to include codeql-cli-2.17.1 --- .../codeql-changelog/codeql-cli-2.17.1.rst | 106 ++++++++++++++++++ .../codeql-changelog/index.rst | 1 + 2 files changed, 107 insertions(+) create mode 100644 docs/codeql/codeql-overview/codeql-changelog/codeql-cli-2.17.1.rst diff --git a/docs/codeql/codeql-overview/codeql-changelog/codeql-cli-2.17.1.rst b/docs/codeql/codeql-overview/codeql-changelog/codeql-cli-2.17.1.rst new file mode 100644 index 00000000000..e0e09d2a227 --- /dev/null +++ b/docs/codeql/codeql-overview/codeql-changelog/codeql-cli-2.17.1.rst @@ -0,0 +1,106 @@ +.. _codeql-cli-2.17.1: + +========================== +CodeQL 2.17.1 (2024-04-24) +========================== + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: none + +This is an overview of changes in the CodeQL CLI and relevant CodeQL query and library packs. For additional updates on changes to the CodeQL code scanning experience, check out the `code scanning section on the GitHub blog `__, `relevant GitHub Changelog updates `__, `changes in the CodeQL extension for Visual Studio Code `__, and the `CodeQL Action changelog `__. + +Security Coverage +----------------- + +CodeQL 2.17.1 runs a total of 412 security queries when configured with the Default suite (covering 160 CWE). The Extended suite enables an additional 130 queries (covering 34 more CWE). 2 security queries have been added with this release. + +CodeQL CLI +---------- + +Deprecations +~~~~~~~~~~~~ + +* The :code:`--mode` option and :code:`-m` alias to :code:`codeql database create`, + :code:`codeql database cleanup`, and :code:`codeql dataset cleanup` has been deprecated. Instead, use the new :code:`--cache-cleanup` option, which has identical behavior. + +Improvements +~~~~~~~~~~~~ + +* Improved the diagnostic message produced when no code is processed when creating a database. If a build mode was specified using + :code:`--build-mode`, the message is now tailored to your build mode. + +Miscellaneous +~~~~~~~~~~~~~ + +* The :code:`scc` tool used by the CodeQL CLI to calculate source code baseline information has been updated to version `3.2.0 `__. + +Query Packs +----------- + +Minor Analysis Improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Java +"""" + +* The :code:`java/unknown-javadoc-parameter` now accepts :code:`@param` tags that apply to the parameters of a record. + +JavaScript/TypeScript +""""""""""""""""""""" + +* :code:`API::Node#getInstance()` now includes instances of subclasses, include transitive subclasses. + The same changes applies to uses of the :code:`Instance` token in data extensions. + +New Queries +~~~~~~~~~~~ + +Ruby +"""" + +* Added a new query, :code:`rb/insecure-mass-assignment`, for finding instances of mass assignment operations accepting arbitrary parameters from remote user input. +* Added a new query, :code:`rb/csrf-protection-not-enabled`, to detect cases where Cross-Site Request Forgery protection is not enabled in Ruby on Rails controllers. + +Language Libraries +------------------ + +Minor Analysis Improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +C# +"" + +* Extracting suppress nullable warning expressions did not work when applied directly to a method call (like :code:`System.Console.Readline()!`). This has been fixed. + +Golang +"""""" + +* Data flow through variables declared in statements of the form :code:`x := y.(type)` at the beginning of type switches has been fixed, which may result in more alerts. +* Added strings.ReplaceAll, http.ParseMultipartForm sanitizers and remove path sanitizer. + +Java +"""" + +* About 6,700 summary models and 6,800 neutral summary models for the JDK that were generated using data flow have been added. This may lead to new alerts being reported. + +Python +"""""" + +* Improved the type-tracking capabilities (and therefore also API graphs) to allow tracking items in tuples and dictionaries. + +Shared Libraries +---------------- + +New Features +~~~~~~~~~~~~ + +Dataflow Analysis +""""""""""""""""" + +* The :code:`PathGraph` result of a data flow computation has been augmented with model provenance information for each of the flow steps. Any qltests that include the edges relation in their output (for example, :code:`.qlref`\ s that reference path-problem queries) will need to be have their expected output updated accordingly. + +Type-flow Analysis +"""""""""""""""""" + +* Initial release. Adds a library to implement type-flow analysis. diff --git a/docs/codeql/codeql-overview/codeql-changelog/index.rst b/docs/codeql/codeql-overview/codeql-changelog/index.rst index 222e133ed19..3662d5fccff 100644 --- a/docs/codeql/codeql-overview/codeql-changelog/index.rst +++ b/docs/codeql/codeql-overview/codeql-changelog/index.rst @@ -11,6 +11,7 @@ A list of queries for each suite and language `is available here Date: Thu, 25 Apr 2024 14:52:36 +0100 Subject: [PATCH 39/39] C++: Fix typo. --- .../CWE/CWE-125/DangerousWorksWithMultibyteOrWideCharacters.ql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/ql/src/experimental/Security/CWE/CWE-125/DangerousWorksWithMultibyteOrWideCharacters.ql b/cpp/ql/src/experimental/Security/CWE/CWE-125/DangerousWorksWithMultibyteOrWideCharacters.ql index 12a44be0d1f..6529bf6cdf8 100644 --- a/cpp/ql/src/experimental/Security/CWE/CWE-125/DangerousWorksWithMultibyteOrWideCharacters.ql +++ b/cpp/ql/src/experimental/Security/CWE/CWE-125/DangerousWorksWithMultibyteOrWideCharacters.ql @@ -24,7 +24,7 @@ predicate exprMayBeString(Expr exp) { fctmp.getAnArgument().(VariableAccess).getTarget() = exp.(VariableAccess).getTarget() or globalValueNumber(fctmp.getAnArgument()) = globalValueNumber(exp) ) and - fctmp.getTarget().hasName(["strlen", "strcat", "strncat", "strcpy", "sptintf", "printf"]) + fctmp.getTarget().hasName(["strlen", "strcat", "strncat", "strcpy", "sprintf", "printf"]) ) or exists(AssignExpr astmp |