Merge branch 'main' of github.com:github/codeql into htmlReg

python/ql/lib/semmle/python/ApiGraphs.qll

@@ -55,7 +55,7 @@ module API {
     /**
      * Gets a call to the function represented by this API component.
      */
-    DataFlow::CallCfgNode getACall() { result = getReturn().getAnImmediateUse() }
+    DataFlow::CallCfgNode getACall() { result = this.getReturn().getAnImmediateUse() }
 
     /**
      * Gets a node representing member `m` of this API component.
@@ -67,21 +67,21 @@ module API {
      */
     bindingset[m]
     bindingset[result]
-    Node getMember(string m) { result = getASuccessor(Label::member(m)) }
+    Node getMember(string m) { result = this.getASuccessor(Label::member(m)) }
 
     /**
      * Gets a node representing a member of this API component where the name of the member is
      * not known statically.
      */
-    Node getUnknownMember() { result = getASuccessor(Label::unknownMember()) }
+    Node getUnknownMember() { result = this.getASuccessor(Label::unknownMember()) }
 
     /**
      * Gets a node representing a member of this API component where the name of the member may
      * or may not be known statically.
      */
     Node getAMember() {
-      result = getASuccessor(Label::member(_)) or
-      result = getUnknownMember()
+      result = this.getASuccessor(Label::member(_)) or
+      result = this.getUnknownMember()
     }
 
     /**
@@ -90,23 +90,25 @@ module API {
      * This predicate may have multiple results when there are multiple invocations of this API component.
      * Consider using `getACall()` if there is a need to distinguish between individual calls.
      */
-    Node getReturn() { result = getASuccessor(Label::return()) }
+    Node getReturn() { result = this.getASuccessor(Label::return()) }
 
     /**
      * Gets a node representing a subclass of the class represented by this node.
      */
-    Node getASubclass() { result = getASuccessor(Label::subclass()) }
+    Node getASubclass() { result = this.getASuccessor(Label::subclass()) }
 
     /**
      * Gets a node representing the result from awaiting this node.
      */
-    Node getAwaited() { result = getASuccessor(Label::await()) }
+    Node getAwaited() { result = this.getASuccessor(Label::await()) }
 
     /**
      * Gets a string representation of the lexicographically least among all shortest access paths
      * from the root to this node.
      */
-    string getPath() { result = min(string p | p = getAPath(Impl::distanceFromRoot(this)) | p) }
+    string getPath() {
+      result = min(string p | p = this.getAPath(Impl::distanceFromRoot(this)) | p)
+    }
 
     /**
      * Gets a node such that there is an edge in the API graph between this node and the other
@@ -124,13 +126,13 @@ module API {
      * Gets a node such that there is an edge in the API graph between this node and the other
      * one.
      */
-    Node getAPredecessor() { result = getAPredecessor(_) }
+    Node getAPredecessor() { result = this.getAPredecessor(_) }
 
     /**
      * Gets a node such that there is an edge in the API graph between that other node and
      * this one.
      */
-    Node getASuccessor() { result = getASuccessor(_) }
+    Node getASuccessor() { result = this.getASuccessor(_) }
 
     /**
      * Gets the data-flow node that gives rise to this node, if any.
@@ -142,16 +144,16 @@ module API {
      * The location spans column `startcolumn` of line `startline` to
      * column `endcolumn` of line `endline` in file `filepath`.
      * For more information, see
-     * [Locations](https://help.semmle.com/QL/learn-ql/locations.html).
+     * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
      */
     predicate hasLocationInfo(
       string filepath, int startline, int startcolumn, int endline, int endcolumn
     ) {
-      getInducingNode().hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+      this.getInducingNode().hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
       or
       // For nodes that do not have a meaningful location, `path` is the empty string and all other
       // parameters are zero.
-      not exists(getInducingNode()) and
+      not exists(this.getInducingNode()) and
       filepath = "" and
       startline = 0 and
       startcolumn = 0 and
@@ -202,7 +204,7 @@ module API {
         or
         this = Impl::MkModuleImport(_) and type = "ModuleImport "
       |
-        result = type + getPath()
+        result = type + this.getPath()
         or
         not exists(this.getPath()) and result = type + "with no path"
       )

python/ql/lib/semmle/python/Comment.qll

@@ -67,7 +67,7 @@ class CommentBlock extends @py_comment {
    * The location spans column `startcolumn` of line `startline` to
    * column `endcolumn` of line `endline` in file `filepath`.
    * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn

python/ql/lib/semmle/python/Concepts.qll

@@ -355,6 +355,53 @@ module SqlExecution {
   }
 }
 
+/**
+ * A data-flow node that executes a regular expression.
+ *
+ * Extend this class to refine existing API models. If you want to model new APIs,
+ * extend `RegexExecution::Range` instead.
+ */
+class RegexExecution extends DataFlow::Node {
+  RegexExecution::Range range;
+
+  RegexExecution() { this = range }
+
+  /** Gets the data flow node for the regex being executed by this node. */
+  DataFlow::Node getRegex() { result = range.getRegex() }
+
+  /** Gets a dataflow node for the string to be searched or matched against. */
+  DataFlow::Node getString() { result = range.getString() }
+
+  /**
+   * Gets the name of this regex execution, typically the name of an executing method.
+   * This is used for nice alert messages and should include the module if possible.
+   */
+  string getName() { result = range.getName() }
+}
+
+/** Provides classes for modeling new regular-expression execution APIs. */
+module RegexExecution {
+  /**
+   * A data-flow node that executes a regular expression.
+   *
+   * Extend this class to model new APIs. If you want to refine existing API models,
+   * extend `RegexExecution` instead.
+   */
+  abstract class Range extends DataFlow::Node {
+    /** Gets the data flow node for the regex being executed by this node. */
+    abstract DataFlow::Node getRegex();
+
+    /** Gets a dataflow node for the string to be searched or matched against. */
+    abstract DataFlow::Node getString();
+
+    /**
+     * Gets the name of this regex execution, typically the name of an executing method.
+     * This is used for nice alert messages and should include the module if possible.
+     */
+    abstract string getName();
+  }
+}
+
 /**
  * A data-flow node that escapes meta-characters, which could be used to prevent
  * injection attacks.
@@ -411,6 +458,9 @@ module Escaping {
 
   /** Gets the escape-kind for escaping a string so it can safely be included in HTML. */
   string getHtmlKind() { result = "html" }
+
+  /** Gets the escape-kind for escaping a string so it can safely be included in HTML. */
+  string getRegexKind() { result = "regex" }
   // TODO: If adding an XML kind, update the modeling of the `MarkupSafe` PyPI package.
   //
   // Technically it claims to escape for both HTML and XML, but for now we don't have
@@ -427,6 +477,14 @@ class HtmlEscaping extends Escaping {
   HtmlEscaping() { range.getKind() = Escaping::getHtmlKind() }
 }
 
+/**
+ * An escape of a string so it can be safely included in
+ * the body of a regex.
+ */
+class RegexEscaping extends Escaping {
+  RegexEscaping() { range.getKind() = Escaping::getRegexKind() }
+}
+
 /** Provides classes for modeling HTTP-related APIs. */
 module HTTP {
   import semmle.python.web.HttpConstants

python/ql/lib/semmle/python/Exprs.qll

@@ -17,7 +17,7 @@ class Expr extends Expr_, AstNode {
    * Whether this expression defines variable `v`
    * If doing dataflow, then consider using SsaVariable.getDefinition() for more precision.
    */
-  predicate defines(Variable v) { this.getASubExpression+().defines(v) }
+  predicate defines(Variable v) { this.getASubExpression().defines(v) }
 
   /** Whether this expression may have a side effect (as determined purely from its syntax) */
   predicate hasSideEffects() {
@@ -240,7 +240,7 @@ class Call extends Call_ {
   /** Gets the tuple (*) argument of this call, provided there is exactly one. */
   Expr getStarArg() {
     count(this.getStarargs()) < 2 and
-    result = getStarargs()
+    result = this.getStarargs()
   }
 }
 

python/ql/lib/semmle/python/Files.qll

@@ -13,7 +13,7 @@ class File extends Container, @file {
    * The location spans column `startcolumn` of line `startline` to
    * column `endcolumn` of line `endline` in file `filepath`.
    * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn
@@ -123,7 +123,7 @@ class Folder extends Container, @folder {
    * The location spans column `startcolumn` of line `startline` to
    * column `endcolumn` of line `endline` in file `filepath`.
    * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn
@@ -256,7 +256,7 @@ abstract class Container extends @container {
    * </table>
    */
   string getBaseName() {
-    result = getAbsolutePath().regexpCapture(".*/(([^/]*?)(?:\\.([^.]*))?)", 1)
+    result = this.getAbsolutePath().regexpCapture(".*/(([^/]*?)(?:\\.([^.]*))?)", 1)
   }
 
   /**
@@ -282,7 +282,9 @@ abstract class Container extends @container {
    * <tr><td>"/tmp/x.tar.gz"</td><td>"gz"</td></tr>
    * </table>
    */
-  string getExtension() { result = getAbsolutePath().regexpCapture(".*/([^/]*?)(\\.([^.]*))?", 3) }
+  string getExtension() {
+    result = this.getAbsolutePath().regexpCapture(".*/([^/]*?)(\\.([^.]*))?", 3)
+  }
 
   /**
    * Gets the stem of this container, that is, the prefix of its base name up to
@@ -301,7 +303,9 @@ abstract class Container extends @container {
    * <tr><td>"/tmp/x.tar.gz"</td><td>"x.tar"</td></tr>
    * </table>
    */
-  string getStem() { result = getAbsolutePath().regexpCapture(".*/([^/]*?)(?:\\.([^.]*))?", 1) }
+  string getStem() {
+    result = this.getAbsolutePath().regexpCapture(".*/([^/]*?)(?:\\.([^.]*))?", 1)
+  }
 
   File getFile(string baseName) {
     result = this.getAFile() and
@@ -323,7 +327,7 @@ abstract class Container extends @container {
   /**
    * Gets a URL representing the location of this container.
    *
-   * For more information see [Providing URLs](https://help.semmle.com/QL/learn-ql/ql/locations.html#providing-urls).
+   * For more information see [Providing URLs](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/#providing-urls).
    */
   abstract string getURL();
 
@@ -429,7 +433,7 @@ class Location extends @location {
    * The location spans column `startcolumn` of line `startline` to
    * column `endcolumn` of line `endline` in file `filepath`.
    * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn
@@ -457,7 +461,7 @@ class Line extends @py_line {
    * The location spans column `startcolumn` of line `startline` to
    * column `endcolumn` of line `endline` in file `filepath`.
    * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn

python/ql/lib/semmle/python/Flow.qll

@@ -851,9 +851,9 @@ class ForNode extends ControlFlowNode {
 
   /** Holds if this `for` statement causes iteration over `sequence` storing each step of the iteration in `target` */
   predicate iterates(ControlFlowNode target, ControlFlowNode sequence) {
-    sequence = getSequence() and
-    target = possibleTarget() and
-    not target = unrolledSuffix().possibleTarget()
+    sequence = this.getSequence() and
+    target = this.possibleTarget() and
+    not target = this.unrolledSuffix().possibleTarget()
   }
 
   /** Gets the sequence node for this `for` statement. */
@@ -1111,7 +1111,7 @@ class BasicBlock extends @py_flow_node {
    * The location spans column `startcolumn` of line `startline` to
    * column `endcolumn` of line `endline` in file `filepath`.
    * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn

python/ql/lib/semmle/python/Function.qll

@@ -58,6 +58,7 @@ class Function extends Function_, Scope, AstNode {
   /** Gets the name of the nth argument (for simple arguments) */
   string getArgName(int index) { result = this.getArg(index).(Name).getId() }
 
+  /** Gets the parameter of this function with the name `name`. */
   Parameter getArgByName(string name) {
     (
       result = this.getAnArg()

python/ql/lib/semmle/python/GuardedControlFlow.qll

@@ -9,6 +9,7 @@ class ConditionBlock extends BasicBlock {
   }
 
   /** Basic blocks controlled by this condition, i.e. those BBs for which the condition is testIsTrue */
+  pragma[nomagic]
   predicate controls(BasicBlock controlled, boolean testIsTrue) {
     /*
      * For this block to control the block 'controlled' with 'testIsTrue' the following must be true:

python/ql/lib/semmle/python/Import.qll

@@ -31,7 +31,7 @@ class ImportExpr extends ImportExpr_ {
     // relative imports are no longer allowed in Python 3
     major_version() < 3 and
     // and can be explicitly turned off in later versions of Python 2
-    not getEnclosingModule().hasFromFuture("absolute_import")
+    not this.getEnclosingModule().hasFromFuture("absolute_import")
   }
 
   /**
@@ -53,8 +53,8 @@ class ImportExpr extends ImportExpr_ {
    * the name of the topmost module that will be imported.
    */
   private string relativeTopName() {
-    getLevel() = -1 and
-    result = basePackageName(1) + "." + this.getTopName() and
+    this.getLevel() = -1 and
+    result = this.basePackageName(1) + "." + this.getTopName() and
     valid_module_name(result)
   }
 
@@ -62,7 +62,7 @@ class ImportExpr extends ImportExpr_ {
     if this.getLevel() <= 0
     then result = this.getTopName()
     else (
-      result = basePackageName(this.getLevel()) and
+      result = this.basePackageName(this.getLevel()) and
       valid_module_name(result)
     )
   }
@@ -73,17 +73,17 @@ class ImportExpr extends ImportExpr_ {
    * which may not be the name of the module.
    */
   string bottomModuleName() {
-    result = relativeTopName() + this.remainderOfName()
+    result = this.relativeTopName() + this.remainderOfName()
     or
-    not exists(relativeTopName()) and
+    not exists(this.relativeTopName()) and
     result = this.qualifiedTopName() + this.remainderOfName()
   }
 
   /** Gets the name of topmost module or package being imported */
   string topModuleName() {
-    result = relativeTopName()
+    result = this.relativeTopName()
     or
-    not exists(relativeTopName()) and
+    not exists(this.relativeTopName()) and
     result = this.qualifiedTopName()
   }
 
@@ -94,7 +94,7 @@ class ImportExpr extends ImportExpr_ {
    */
   string getImportedModuleName() {
     exists(string bottomName | bottomName = this.bottomModuleName() |
-      if this.isTop() then result = topModuleName() else result = bottomName
+      if this.isTop() then result = this.topModuleName() else result = bottomName
     )
   }
 

python/ql/lib/semmle/python/Module.qll

@@ -86,13 +86,13 @@ class Module extends Module_, Scope, AstNode {
   /** Gets the package containing this module (or parent package if this is a package) */
   Module getPackage() {
     this.getName().matches("%.%") and
-    result.getName() = getName().regexpReplaceAll("\\.[^.]*$", "")
+    result.getName() = this.getName().regexpReplaceAll("\\.[^.]*$", "")
   }
 
   /** Gets the name of the package containing this module */
   string getPackageName() {
     this.getName().matches("%.%") and
-    result = getName().regexpReplaceAll("\\.[^.]*$", "")
+    result = this.getName().regexpReplaceAll("\\.[^.]*$", "")
   }
 
   /** Gets the metrics for this module */

python/ql/lib/semmle/python/PrintAst.qll

@@ -52,8 +52,7 @@ private newtype TPrintAstNode =
   TStmtListNode(StmtList list) {
     shouldPrint(list.getAnItem(), _) and
     not list = any(Module mod).getBody() and
-    not forall(AstNode child | child = list.getAnItem() | isNotNeeded(child)) and
-    exists(list.getAnItem())
+    not forall(AstNode child | child = list.getAnItem() | isNotNeeded(child))
   } or
   TRegExpTermNode(RegExpTerm term) {
     exists(StrConst str | term.getRootTerm() = getParsedRegExp(str) and shouldPrint(str, _))

python/ql/lib/semmle/python/RegexTreeView.qll

@@ -49,16 +49,17 @@ newtype TRegExpParent =
  * or another regular expression term.
  */
 class RegExpParent extends TRegExpParent {
+  /** Gets a textual representation of this element. */
   string toString() { result = "RegExpParent" }
 
   /** Gets the `i`th child term. */
   abstract RegExpTerm getChild(int i);
 
   /** Gets a child term . */
-  RegExpTerm getAChild() { result = getChild(_) }
+  RegExpTerm getAChild() { result = this.getChild(_) }
 
   /** Gets the number of child terms. */
-  int getNumChild() { result = count(getAChild()) }
+  int getNumChild() { result = count(this.getAChild()) }
 
   /** Gets the associated regex. */
   abstract Regex getRegex();
@@ -72,14 +73,18 @@ class RegExpLiteral extends TRegExpLiteral, RegExpParent {
 
   override RegExpTerm getChild(int i) { i = 0 and result.getRegex() = re and result.isRootTerm() }
 
+  /** Holds if dot, `.`, matches all characters, including newlines. */
   predicate isDotAll() { re.getAMode() = "DOTALL" }
 
+  /** Holds if this regex matching is case-insensitive for this regex. */
   predicate isIgnoreCase() { re.getAMode() = "IGNORECASE" }
 
+  /** Get a string representing all modes for this regex. */
   string getFlags() { result = concat(string mode | mode = re.getAMode() | mode, " | ") }
 
   override Regex getRegex() { result = re }
 
+  /** Gets the primary QL class for this regex. */
   string getPrimaryQLClass() { result = "RegExpLiteral" }
 }
 
@@ -117,7 +122,7 @@ class RegExpTerm extends RegExpParent {
   RegExpTerm getRootTerm() {
     this.isRootTerm() and result = this
     or
-    result = getParent().(RegExpTerm).getRootTerm()
+    result = this.getParent().(RegExpTerm).getRootTerm()
   }
 
   /**
@@ -196,7 +201,7 @@ class RegExpTerm extends RegExpParent {
 
   /** Gets the regular expression term that is matched (textually) before this one, if any. */
   RegExpTerm getPredecessor() {
-    exists(RegExpTerm parent | parent = getParent() |
+    exists(RegExpTerm parent | parent = this.getParent() |
       result = parent.(RegExpSequence).previousElement(this)
       or
       not exists(parent.(RegExpSequence).previousElement(this)) and
@@ -207,7 +212,7 @@ class RegExpTerm extends RegExpParent {
 
   /** Gets the regular expression term that is matched (textually) after this one, if any. */
   RegExpTerm getSuccessor() {
-    exists(RegExpTerm parent | parent = getParent() |
+    exists(RegExpTerm parent | parent = this.getParent() |
       result = parent.(RegExpSequence).nextElement(this)
       or
       not exists(parent.(RegExpSequence).nextElement(this)) and
@@ -246,8 +251,10 @@ class RegExpQuantifier extends RegExpTerm, TRegExpQuantifier {
     result.getEnd() = part_end
   }
 
+  /** Hols if this term may match an unlimited number of times. */
   predicate mayRepeatForever() { may_repeat_forever = true }
 
+  /** Gets the qualifier for this term. That is e.g "?" for "a?". */
   string getQualifier() { result = re.getText().substring(part_end, end) }
 
   override string getPrimaryQLClass() { result = "RegExpQuantifier" }
@@ -322,8 +329,10 @@ class RegExpRange extends RegExpQuantifier {
 
   RegExpRange() { re.multiples(part_end, end, lower, upper) }
 
+  /** Gets the string defining the upper bound of this range, if any. */
   string getUpper() { result = upper }
 
+  /** Gets the string defining the lower bound of this range, if any. */
   string getLower() { result = lower }
 
   /**
@@ -358,7 +367,7 @@ class RegExpSequence extends RegExpTerm, TRegExpSequence {
   override RegExpTerm getChild(int i) { result = seqChild(re, start, end, i) }
 
   /** Gets the element preceding `element` in this sequence. */
-  RegExpTerm previousElement(RegExpTerm element) { element = nextElement(result) }
+  RegExpTerm previousElement(RegExpTerm element) { element = this.nextElement(result) }
 
   /** Gets the element following `element` in this sequence. */
   RegExpTerm nextElement(RegExpTerm element) {
@@ -462,15 +471,17 @@ class RegExpEscape extends RegExpNormalChar {
     or
     this.getUnescaped() = "v" and result = 11.toUnicode()
     or
-    isUnicode() and
-    result = getUnicode()
+    this.isUnicode() and
+    result = this.getUnicode()
   }
 
+  /** Holds if this terms name is given by the part following the escape character. */
   predicate isIdentityEscape() { not this.getUnescaped() in ["n", "r", "t", "f"] }
 
   override string getPrimaryQLClass() { result = "RegExpEscape" }
 
-  string getUnescaped() { result = this.getText().suffix(1) }
+  /** Gets the part of the term following the escape character. That is e.g. "w" if the term is "\w". */
+  private string getUnescaped() { result = this.getText().suffix(1) }
 
   /**
    * Gets the text for this escape. That is e.g. "\w".
@@ -480,7 +491,7 @@ class RegExpEscape extends RegExpNormalChar {
   /**
    * Holds if this is a unicode escape.
    */
-  private predicate isUnicode() { getText().prefix(2) = ["\\u", "\\U"] }
+  private predicate isUnicode() { this.getText().prefix(2) = ["\\u", "\\U"] }
 
   /**
    * Gets the unicode char for this escape.
@@ -544,15 +555,8 @@ class RegExpWordBoundary extends RegExpEscape {
  * ```
  */
 class RegExpCharacterClassEscape extends RegExpEscape {
-  // string value;
-  RegExpCharacterClassEscape() {
-    // value = re.getText().substring(start + 1, end) and
-    // value in ["d", "D", "s", "S", "w", "W"]
-    this.getValue() in ["d", "D", "s", "S", "w", "W"]
-  }
+  RegExpCharacterClassEscape() { this.getValue() in ["d", "D", "s", "S", "w", "W"] }
 
-  /** Gets the name of the character class; for example, `w` for `\w`. */
-  // override string getValue() { result = value }
   override RegExpTerm getChild(int i) { none() }
 
   override string getPrimaryQLClass() { result = "RegExpCharacterClassEscape" }
@@ -571,19 +575,22 @@ class RegExpCharacterClassEscape extends RegExpEscape {
 class RegExpCharacterClass extends RegExpTerm, TRegExpCharacterClass {
   RegExpCharacterClass() { this = TRegExpCharacterClass(re, start, end) }
 
+  /** Holds if this character class is inverted, matching the opposite of its content. */
   predicate isInverted() { re.getChar(start + 1) = "^" }
 
+  /** Gets the `i`th char inside this charater class. */
   string getCharThing(int i) { result = re.getChar(i + start) }
 
+  /** Holds if this character class can match anything. */
   predicate isUniversalClass() {
     // [^]
-    isInverted() and not exists(getAChild())
+    this.isInverted() and not exists(this.getAChild())
     or
     // [\w\W] and similar
-    not isInverted() and
+    not this.isInverted() and
     exists(string cce1, string cce2 |
-      cce1 = getAChild().(RegExpCharacterClassEscape).getValue() and
-      cce2 = getAChild().(RegExpCharacterClassEscape).getValue()
+      cce1 = this.getAChild().(RegExpCharacterClassEscape).getValue() and
+      cce2 = this.getAChild().(RegExpCharacterClassEscape).getValue()
     |
       cce1 != cce2 and cce1.toLowerCase() = cce2.toLowerCase()
     )
@@ -628,6 +635,7 @@ class RegExpCharacterRange extends RegExpTerm, TRegExpCharacterRange {
     re.charRange(_, start, lower_end, upper_start, end)
   }
 
+  /** Holds if this range goes from `lo` to `hi`, in effect is `lo-hi`. */
   predicate isRange(string lo, string hi) {
     lo = re.getText().substring(start, lower_end) and
     hi = re.getText().substring(upper_start, end)
@@ -661,8 +669,13 @@ class RegExpCharacterRange extends RegExpTerm, TRegExpCharacterRange {
 class RegExpNormalChar extends RegExpTerm, TRegExpNormalChar {
   RegExpNormalChar() { this = TRegExpNormalChar(re, start, end) }
 
+  /**
+   * Holds if this constant represents a valid Unicode character (as opposed
+   * to a surrogate code point that does not correspond to a character by itself.)
+   */
   predicate isCharacter() { any() }
 
+  /** Gets the string representation of the char matched by this term. */
   string getValue() { result = re.getText().substring(start, end) }
 
   override RegExpTerm getChild(int i) { none() }
@@ -692,15 +705,15 @@ class RegExpConstant extends RegExpTerm {
       qstart <= start and end <= qend
     ) and
     value = this.(RegExpNormalChar).getValue()
-    // This will never hold
-    // or
-    // this = TRegExpSpecialChar(re, start, end) and
-    // re.inCharSet(start) and
-    // value = this.(RegExpSpecialChar).getChar()
   }
 
+  /**
+   * Holds if this constant represents a valid Unicode character (as opposed
+   * to a surrogate code point that does not correspond to a character by itself.)
+   */
   predicate isCharacter() { any() }
 
+  /** Gets the string matched by this constant term. */
   string getValue() { result = value }
 
   override RegExpTerm getChild(int i) { none() }
@@ -739,10 +752,6 @@ class RegExpGroup extends RegExpTerm, TRegExpGroup {
   /** Gets the name of this capture group, if any. */
   string getName() { result = re.getGroupName(start, end) }
 
-  predicate isCharacter() { any() }
-
-  string getValue() { result = re.getText().substring(start, end) }
-
   override RegExpTerm getChild(int i) {
     result.getRegex() = re and
     i = 0 and
@@ -770,8 +779,13 @@ class RegExpSpecialChar extends RegExpTerm, TRegExpSpecialChar {
     re.specialCharacter(start, end, char)
   }
 
+  /**
+   * Holds if this constant represents a valid Unicode character (as opposed
+   * to a surrogate code point that does not correspond to a character by itself.)
+   */
   predicate isCharacter() { any() }
 
+  /** Gets the char for this term. */
   string getChar() { result = char }
 
   override RegExpTerm getChild(int i) { none() }
@@ -836,8 +850,6 @@ class RegExpCaret extends RegExpSpecialChar {
 class RegExpZeroWidthMatch extends RegExpGroup {
   RegExpZeroWidthMatch() { re.zeroWidthMatch(start, end) }
 
-  override predicate isCharacter() { any() }
-
   override RegExpTerm getChild(int i) { none() }
 
   override string getPrimaryQLClass() { result = "RegExpZeroWidthMatch" }

python/ql/lib/semmle/python/concepts/CryptoAlgorithms.qll

@@ -15,68 +15,35 @@
  */
 private module AlgorithmNames {
   predicate isStrongHashingAlgorithm(string name) {
-    name = "DSA" or
-    name = "ED25519" or
-    name = "ES256" or
-    name = "ECDSA256" or
-    name = "ES384" or
-    name = "ECDSA384" or
-    name = "ES512" or
-    name = "ECDSA512" or
-    name = "SHA2" or
-    name = "SHA224" or
-    name = "SHA256" or
-    name = "SHA384" or
-    name = "SHA512" or
-    name = "SHA3"
+    name =
+      [
+        "DSA", "ED25519", "ES256", "ECDSA256", "ES384", "ECDSA384", "ES512", "ECDSA512", "SHA2",
+        "SHA224", "SHA256", "SHA384", "SHA512", "SHA3", "SHA3224", "SHA3256", "SHA3384", "SHA3512"
+      ]
   }
 
   predicate isWeakHashingAlgorithm(string name) {
-    name = "HAVEL128" or
-    name = "MD2" or
-    name = "MD4" or
-    name = "MD5" or
-    name = "PANAMA" or
-    name = "RIPEMD" or
-    name = "RIPEMD128" or
-    name = "RIPEMD256" or
-    name = "RIPEMD160" or
-    name = "RIPEMD320" or
-    name = "SHA0" or
-    name = "SHA1"
+    name =
+      [
+        "HAVEL128", "MD2", "MD4", "MD5", "PANAMA", "RIPEMD", "RIPEMD128", "RIPEMD256", "RIPEMD160",
+        "RIPEMD320", "SHA0", "SHA1"
+      ]
   }
 
   predicate isStrongEncryptionAlgorithm(string name) {
-    name = "AES" or
-    name = "AES128" or
-    name = "AES192" or
-    name = "AES256" or
-    name = "AES512" or
-    name = "RSA" or
-    name = "RABBIT" or
-    name = "BLOWFISH"
+    name = ["AES", "AES128", "AES192", "AES256", "AES512", "RSA", "RABBIT", "BLOWFISH"]
   }
 
   predicate isWeakEncryptionAlgorithm(string name) {
-    name = "DES" or
-    name = "3DES" or
-    name = "TRIPLEDES" or
-    name = "TDEA" or
-    name = "TRIPLEDEA" or
-    name = "ARC2" or
-    name = "RC2" or
-    name = "ARC4" or
-    name = "RC4" or
-    name = "ARCFOUR" or
-    name = "ARC5" or
-    name = "RC5"
+    name =
+      [
+        "DES", "3DES", "TRIPLEDES", "TDEA", "TRIPLEDEA", "ARC2", "RC2", "ARC4", "RC4", "ARCFOUR",
+        "ARC5", "RC5"
+      ]
   }
 
   predicate isStrongPasswordHashingAlgorithm(string name) {
-    name = "ARGON2" or
-    name = "PBKDF2" or
-    name = "BCRYPT" or
-    name = "SCRYPT"
+    name = ["ARGON2", "PBKDF2", "BCRYPT", "SCRYPT"]
   }
 
   predicate isWeakPasswordHashingAlgorithm(string name) { none() }

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll

@@ -110,12 +110,12 @@ abstract class Configuration extends string {
   /**
    * Holds if data may flow from some source to `sink` for this configuration.
    */
-  predicate hasFlowTo(Node sink) { hasFlow(_, sink) }
+  predicate hasFlowTo(Node sink) { this.hasFlow(_, sink) }
 
   /**
    * Holds if data may flow from some source to `sink` for this configuration.
    */
-  predicate hasFlowToExpr(DataFlowExpr sink) { hasFlowTo(exprNode(sink)) }
+  predicate hasFlowToExpr(DataFlowExpr sink) { this.hasFlowTo(exprNode(sink)) }
 
   /**
    * Gets the exploration limit for `hasPartialFlow` and `hasPartialFlowRev`
@@ -244,6 +244,8 @@ private class ParamNodeEx extends NodeEx {
   }
 
   int getPosition() { this.isParameterOf(_, result) }
+
+  predicate allowParameterReturnInSelf() { allowParameterReturnInSelfCached(this.asNode()) }
 }
 
 private class RetNodeEx extends NodeEx {
@@ -744,8 +746,12 @@ private module Stage1 {
       returnFlowCallableNodeCand(c, kind, config) and
       p.getEnclosingCallable() = c and
       exists(ap) and
-      // we don't expect a parameter to return stored in itself
-      not kind.(ParamUpdateReturnKind).getPosition() = p.getPosition()
+      // we don't expect a parameter to return stored in itself, unless explicitly allowed
+      (
+        not kind.(ParamUpdateReturnKind).getPosition() = p.getPosition()
+        or
+        p.allowParameterReturnInSelf()
+      )
     )
   }
 
@@ -1394,8 +1400,12 @@ private module Stage2 {
       fwdFlow(ret, any(CcCall ccc), apSome(ap), ap0, config) and
       kind = ret.getKind() and
       p.getPosition() = pos and
-      // we don't expect a parameter to return stored in itself
-      not kind.(ParamUpdateReturnKind).getPosition() = pos
+      // we don't expect a parameter to return stored in itself, unless explicitly allowed
+      (
+        not kind.(ParamUpdateReturnKind).getPosition() = pos
+        or
+        p.allowParameterReturnInSelf()
+      )
     )
   }
 
@@ -2083,8 +2093,12 @@ private module Stage3 {
       fwdFlow(ret, any(CcCall ccc), apSome(ap), ap0, config) and
       kind = ret.getKind() and
       p.getPosition() = pos and
-      // we don't expect a parameter to return stored in itself
-      not kind.(ParamUpdateReturnKind).getPosition() = pos
+      // we don't expect a parameter to return stored in itself, unless explicitly allowed
+      (
+        not kind.(ParamUpdateReturnKind).getPosition() = pos
+        or
+        p.allowParameterReturnInSelf()
+      )
     )
   }
 
@@ -2139,7 +2153,8 @@ private predicate expensiveLen2unfolding(TypedContent tc, Configuration config)
       ) and
     accessPathApproxCostLimits(apLimit, tupleLimit) and
     apLimit < tails and
-    tupleLimit < (tails - 1) * nodes
+    tupleLimit < (tails - 1) * nodes and
+    not tc.forceHighPrecision()
   )
 }
 
@@ -2842,8 +2857,12 @@ private module Stage4 {
       fwdFlow(ret, any(CcCall ccc), apSome(ap), ap0, config) and
       kind = ret.getKind() and
       p.getPosition() = pos and
-      // we don't expect a parameter to return stored in itself
-      not kind.(ParamUpdateReturnKind).getPosition() = pos
+      // we don't expect a parameter to return stored in itself, unless explicitly allowed
+      (
+        not kind.(ParamUpdateReturnKind).getPosition() = pos
+        or
+        p.allowParameterReturnInSelf()
+      )
     )
   }
 
@@ -2916,6 +2935,8 @@ private class SummaryCtxSome extends SummaryCtx, TSummaryCtxSome {
 
   int getParameterPos() { p.isParameterOf(_, result) }
 
+  ParamNodeEx getParamNode() { result = p }
+
   override string toString() { result = p + ": " + ap }
 
   predicate hasLocationInfo(
@@ -2973,12 +2994,15 @@ private AccessPathApprox getATail(AccessPathApprox apa, Configuration config) {
  * expected to be expensive. Holds with `unfold = true` otherwise.
  */
 private predicate evalUnfold(AccessPathApprox apa, boolean unfold, Configuration config) {
-  exists(int aps, int nodes, int apLimit, int tupleLimit |
-    aps = countPotentialAps(apa, config) and
-    nodes = countNodesUsingAccessPath(apa, config) and
-    accessPathCostLimits(apLimit, tupleLimit) and
-    if apLimit < aps and tupleLimit < (aps - 1) * nodes then unfold = false else unfold = true
-  )
+  if apa.getHead().forceHighPrecision()
+  then unfold = true
+  else
+    exists(int aps, int nodes, int apLimit, int tupleLimit |
+      aps = countPotentialAps(apa, config) and
+      nodes = countNodesUsingAccessPath(apa, config) and
+      accessPathCostLimits(apLimit, tupleLimit) and
+      if apLimit < aps and tupleLimit < (aps - 1) * nodes then unfold = false else unfold = true
+    )
 }
 
 /**
@@ -3166,7 +3190,7 @@ private class AccessPathCons extends AccessPath, TAccessPathCons {
   }
 
   override string toString() {
-    result = "[" + this.toStringImpl(true) + length().toString() + ")]"
+    result = "[" + this.toStringImpl(true) + this.length().toString() + ")]"
     or
     result = "[" + this.toStringImpl(false)
   }
@@ -3248,7 +3272,7 @@ class PathNode extends TPathNode {
    * The location spans column `startcolumn` of line `startline` to
    * column `endcolumn` of line `endline` in file `filepath`.
    * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn
@@ -3305,9 +3329,11 @@ abstract private class PathNodeImpl extends PathNode {
     result = " <" + this.(PathNodeMid).getCallContext().toString() + ">"
   }
 
-  override string toString() { result = this.getNodeEx().toString() + ppAp() }
+  override string toString() { result = this.getNodeEx().toString() + this.ppAp() }
 
-  override string toStringWithContext() { result = this.getNodeEx().toString() + ppAp() + ppCtx() }
+  override string toStringWithContext() {
+    result = this.getNodeEx().toString() + this.ppAp() + this.ppCtx()
+  }
 
   override predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn
@@ -3375,11 +3401,11 @@ private class PathNodeMid extends PathNodeImpl, TPathNodeMid {
 
   override PathNodeImpl getASuccessorImpl() {
     // an intermediate step to another intermediate node
-    result = getSuccMid()
+    result = this.getSuccMid()
     or
     // a final step to a sink via zero steps means we merge the last two steps to prevent trivial-looking edges
     exists(PathNodeMid mid, PathNodeSink sink |
-      mid = getSuccMid() and
+      mid = this.getSuccMid() and
       mid.getNodeEx() = sink.getNodeEx() and
       mid.getAp() instanceof AccessPathNil and
       sink.getConfiguration() = unbindConf(mid.getConfiguration()) and
@@ -3456,7 +3482,7 @@ private predicate pathStep(
   exists(TypedContent tc | pathReadStep(mid, node, ap.push(tc), tc, cc)) and
   sc = mid.getSummaryCtx()
   or
-  pathIntoCallable(mid, node, _, cc, sc, _) and ap = mid.getAp()
+  pathIntoCallable(mid, node, _, cc, sc, _, _) and ap = mid.getAp()
   or
   pathOutOfCallable(mid, node, cc) and ap = mid.getAp() and sc instanceof SummaryCtxNone
   or
@@ -3533,14 +3559,16 @@ private predicate pathOutOfCallable(PathNodeMid mid, NodeEx out, CallContext cc)
  */
 pragma[noinline]
 private predicate pathIntoArg(
-  PathNodeMid mid, int i, CallContext cc, DataFlowCall call, AccessPath ap, AccessPathApprox apa
+  PathNodeMid mid, int i, CallContext cc, DataFlowCall call, AccessPath ap, AccessPathApprox apa,
+  Configuration config
 ) {
   exists(ArgNode arg |
     arg = mid.getNodeEx().asNode() and
     cc = mid.getCallContext() and
     arg.argumentOf(call, i) and
     ap = mid.getAp() and
-    apa = ap.getApprox()
+    apa = ap.getApprox() and
+    config = mid.getConfiguration()
   )
 }
 
@@ -3557,12 +3585,14 @@ private predicate parameterCand(
 pragma[nomagic]
 private predicate pathIntoCallable0(
   PathNodeMid mid, DataFlowCallable callable, int i, CallContext outercc, DataFlowCall call,
-  AccessPath ap
+  AccessPath ap, Configuration config
 ) {
   exists(AccessPathApprox apa |
-    pathIntoArg(mid, i, outercc, call, ap, apa) and
+    pathIntoArg(mid, pragma[only_bind_into](i), outercc, call, ap, pragma[only_bind_into](apa),
+      pragma[only_bind_into](config)) and
     callable = resolveCall(call, outercc) and
-    parameterCand(callable, any(int j | j <= i and j >= i), apa, mid.getConfiguration())
+    parameterCand(callable, pragma[only_bind_into](i), pragma[only_bind_into](apa),
+      pragma[only_bind_into](config))
   )
 }
 
@@ -3571,12 +3601,13 @@ private predicate pathIntoCallable0(
  * before and after entering the callable are `outercc` and `innercc`,
  * respectively.
  */
+pragma[nomagic]
 private predicate pathIntoCallable(
   PathNodeMid mid, ParamNodeEx p, CallContext outercc, CallContextCall innercc, SummaryCtx sc,
-  DataFlowCall call
+  DataFlowCall call, Configuration config
 ) {
   exists(int i, DataFlowCallable callable, AccessPath ap |
-    pathIntoCallable0(mid, callable, i, outercc, call, ap) and
+    pathIntoCallable0(mid, callable, i, outercc, call, ap, config) and
     p.isParameterOf(callable, i) and
     (
       sc = TSummaryCtxSome(p, ap)
@@ -3606,18 +3637,23 @@ private predicate paramFlowsThrough(
     ap = mid.getAp() and
     apa = ap.getApprox() and
     pos = sc.getParameterPos() and
-    not kind.(ParamUpdateReturnKind).getPosition() = pos
+    // we don't expect a parameter to return stored in itself, unless explicitly allowed
+    (
+      not kind.(ParamUpdateReturnKind).getPosition() = pos
+      or
+      sc.getParamNode().allowParameterReturnInSelf()
+    )
   )
 }
 
 pragma[nomagic]
 private predicate pathThroughCallable0(
   DataFlowCall call, PathNodeMid mid, ReturnKindExt kind, CallContext cc, AccessPath ap,
-  AccessPathApprox apa
+  AccessPathApprox apa, Configuration config
 ) {
   exists(CallContext innercc, SummaryCtx sc |
-    pathIntoCallable(mid, _, cc, innercc, sc, call) and
-    paramFlowsThrough(kind, innercc, sc, ap, apa, unbindConf(mid.getConfiguration()))
+    pathIntoCallable(mid, _, cc, innercc, sc, call, config) and
+    paramFlowsThrough(kind, innercc, sc, ap, apa, config)
   )
 }
 
@@ -3627,9 +3663,9 @@ private predicate pathThroughCallable0(
  */
 pragma[noinline]
 private predicate pathThroughCallable(PathNodeMid mid, NodeEx out, CallContext cc, AccessPath ap) {
-  exists(DataFlowCall call, ReturnKindExt kind, AccessPathApprox apa |
-    pathThroughCallable0(call, mid, kind, cc, ap, apa) and
-    out = getAnOutNodeFlow(kind, call, apa, unbindConf(mid.getConfiguration()))
+  exists(DataFlowCall call, ReturnKindExt kind, AccessPathApprox apa, Configuration config |
+    pathThroughCallable0(call, mid, kind, cc, ap, apa, config) and
+    out = getAnOutNodeFlow(kind, call, apa, config)
   )
 }
 
@@ -3643,9 +3679,11 @@ private module Subpaths {
     PathNode arg, ParamNodeEx par, SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind,
     NodeEx out, AccessPath apout
   ) {
-    pathThroughCallable(arg, out, _, apout) and
-    pathIntoCallable(arg, par, _, innercc, sc, _) and
-    paramFlowsThrough(kind, innercc, sc, apout, _, unbindConf(arg.getConfiguration()))
+    exists(Configuration config |
+      pathThroughCallable(arg, out, _, pragma[only_bind_into](apout)) and
+      pathIntoCallable(arg, par, _, innercc, sc, _, config) and
+      paramFlowsThrough(kind, innercc, sc, pragma[only_bind_into](apout), _, unbindConf(config))
+    )
   }
 
   /**
@@ -4032,7 +4070,7 @@ private module FlowExploration {
      * The location spans column `startcolumn` of line `startline` to
      * column `endcolumn` of line `endline` in file `filepath`.
      * For more information, see
-     * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+     * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
      */
     predicate hasLocationInfo(
       string filepath, int startline, int startcolumn, int endline, int endcolumn

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll

@@ -110,12 +110,12 @@ abstract class Configuration extends string {
   /**
    * Holds if data may flow from some source to `sink` for this configuration.
    */
-  predicate hasFlowTo(Node sink) { hasFlow(_, sink) }
+  predicate hasFlowTo(Node sink) { this.hasFlow(_, sink) }
 
   /**
    * Holds if data may flow from some source to `sink` for this configuration.
    */
-  predicate hasFlowToExpr(DataFlowExpr sink) { hasFlowTo(exprNode(sink)) }
+  predicate hasFlowToExpr(DataFlowExpr sink) { this.hasFlowTo(exprNode(sink)) }
 
   /**
    * Gets the exploration limit for `hasPartialFlow` and `hasPartialFlowRev`
@@ -244,6 +244,8 @@ private class ParamNodeEx extends NodeEx {
   }
 
   int getPosition() { this.isParameterOf(_, result) }
+
+  predicate allowParameterReturnInSelf() { allowParameterReturnInSelfCached(this.asNode()) }
 }
 
 private class RetNodeEx extends NodeEx {
@@ -744,8 +746,12 @@ private module Stage1 {
       returnFlowCallableNodeCand(c, kind, config) and
       p.getEnclosingCallable() = c and
       exists(ap) and
-      // we don't expect a parameter to return stored in itself
-      not kind.(ParamUpdateReturnKind).getPosition() = p.getPosition()
+      // we don't expect a parameter to return stored in itself, unless explicitly allowed
+      (
+        not kind.(ParamUpdateReturnKind).getPosition() = p.getPosition()
+        or
+        p.allowParameterReturnInSelf()
+      )
     )
   }
 
@@ -1394,8 +1400,12 @@ private module Stage2 {
       fwdFlow(ret, any(CcCall ccc), apSome(ap), ap0, config) and
       kind = ret.getKind() and
       p.getPosition() = pos and
-      // we don't expect a parameter to return stored in itself
-      not kind.(ParamUpdateReturnKind).getPosition() = pos
+      // we don't expect a parameter to return stored in itself, unless explicitly allowed
+      (
+        not kind.(ParamUpdateReturnKind).getPosition() = pos
+        or
+        p.allowParameterReturnInSelf()
+      )
     )
   }
 
@@ -2083,8 +2093,12 @@ private module Stage3 {
       fwdFlow(ret, any(CcCall ccc), apSome(ap), ap0, config) and
       kind = ret.getKind() and
       p.getPosition() = pos and
-      // we don't expect a parameter to return stored in itself
-      not kind.(ParamUpdateReturnKind).getPosition() = pos
+      // we don't expect a parameter to return stored in itself, unless explicitly allowed
+      (
+        not kind.(ParamUpdateReturnKind).getPosition() = pos
+        or
+        p.allowParameterReturnInSelf()
+      )
     )
   }
 
@@ -2139,7 +2153,8 @@ private predicate expensiveLen2unfolding(TypedContent tc, Configuration config)
       ) and
     accessPathApproxCostLimits(apLimit, tupleLimit) and
     apLimit < tails and
-    tupleLimit < (tails - 1) * nodes
+    tupleLimit < (tails - 1) * nodes and
+    not tc.forceHighPrecision()
   )
 }
 
@@ -2842,8 +2857,12 @@ private module Stage4 {
       fwdFlow(ret, any(CcCall ccc), apSome(ap), ap0, config) and
       kind = ret.getKind() and
       p.getPosition() = pos and
-      // we don't expect a parameter to return stored in itself
-      not kind.(ParamUpdateReturnKind).getPosition() = pos
+      // we don't expect a parameter to return stored in itself, unless explicitly allowed
+      (
+        not kind.(ParamUpdateReturnKind).getPosition() = pos
+        or
+        p.allowParameterReturnInSelf()
+      )
     )
   }
 
@@ -2916,6 +2935,8 @@ private class SummaryCtxSome extends SummaryCtx, TSummaryCtxSome {
 
   int getParameterPos() { p.isParameterOf(_, result) }
 
+  ParamNodeEx getParamNode() { result = p }
+
   override string toString() { result = p + ": " + ap }
 
   predicate hasLocationInfo(
@@ -2973,12 +2994,15 @@ private AccessPathApprox getATail(AccessPathApprox apa, Configuration config) {
  * expected to be expensive. Holds with `unfold = true` otherwise.
  */
 private predicate evalUnfold(AccessPathApprox apa, boolean unfold, Configuration config) {
-  exists(int aps, int nodes, int apLimit, int tupleLimit |
-    aps = countPotentialAps(apa, config) and
-    nodes = countNodesUsingAccessPath(apa, config) and
-    accessPathCostLimits(apLimit, tupleLimit) and
-    if apLimit < aps and tupleLimit < (aps - 1) * nodes then unfold = false else unfold = true
-  )
+  if apa.getHead().forceHighPrecision()
+  then unfold = true
+  else
+    exists(int aps, int nodes, int apLimit, int tupleLimit |
+      aps = countPotentialAps(apa, config) and
+      nodes = countNodesUsingAccessPath(apa, config) and
+      accessPathCostLimits(apLimit, tupleLimit) and
+      if apLimit < aps and tupleLimit < (aps - 1) * nodes then unfold = false else unfold = true
+    )
 }
 
 /**
@@ -3166,7 +3190,7 @@ private class AccessPathCons extends AccessPath, TAccessPathCons {
   }
 
   override string toString() {
-    result = "[" + this.toStringImpl(true) + length().toString() + ")]"
+    result = "[" + this.toStringImpl(true) + this.length().toString() + ")]"
     or
     result = "[" + this.toStringImpl(false)
   }
@@ -3248,7 +3272,7 @@ class PathNode extends TPathNode {
    * The location spans column `startcolumn` of line `startline` to
    * column `endcolumn` of line `endline` in file `filepath`.
    * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn
@@ -3305,9 +3329,11 @@ abstract private class PathNodeImpl extends PathNode {
     result = " <" + this.(PathNodeMid).getCallContext().toString() + ">"
   }
 
-  override string toString() { result = this.getNodeEx().toString() + ppAp() }
+  override string toString() { result = this.getNodeEx().toString() + this.ppAp() }
 
-  override string toStringWithContext() { result = this.getNodeEx().toString() + ppAp() + ppCtx() }
+  override string toStringWithContext() {
+    result = this.getNodeEx().toString() + this.ppAp() + this.ppCtx()
+  }
 
   override predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn
@@ -3375,11 +3401,11 @@ private class PathNodeMid extends PathNodeImpl, TPathNodeMid {
 
   override PathNodeImpl getASuccessorImpl() {
     // an intermediate step to another intermediate node
-    result = getSuccMid()
+    result = this.getSuccMid()
     or
     // a final step to a sink via zero steps means we merge the last two steps to prevent trivial-looking edges
     exists(PathNodeMid mid, PathNodeSink sink |
-      mid = getSuccMid() and
+      mid = this.getSuccMid() and
       mid.getNodeEx() = sink.getNodeEx() and
       mid.getAp() instanceof AccessPathNil and
       sink.getConfiguration() = unbindConf(mid.getConfiguration()) and
@@ -3456,7 +3482,7 @@ private predicate pathStep(
   exists(TypedContent tc | pathReadStep(mid, node, ap.push(tc), tc, cc)) and
   sc = mid.getSummaryCtx()
   or
-  pathIntoCallable(mid, node, _, cc, sc, _) and ap = mid.getAp()
+  pathIntoCallable(mid, node, _, cc, sc, _, _) and ap = mid.getAp()
   or
   pathOutOfCallable(mid, node, cc) and ap = mid.getAp() and sc instanceof SummaryCtxNone
   or
@@ -3533,14 +3559,16 @@ private predicate pathOutOfCallable(PathNodeMid mid, NodeEx out, CallContext cc)
  */
 pragma[noinline]
 private predicate pathIntoArg(
-  PathNodeMid mid, int i, CallContext cc, DataFlowCall call, AccessPath ap, AccessPathApprox apa
+  PathNodeMid mid, int i, CallContext cc, DataFlowCall call, AccessPath ap, AccessPathApprox apa,
+  Configuration config
 ) {
   exists(ArgNode arg |
     arg = mid.getNodeEx().asNode() and
     cc = mid.getCallContext() and
     arg.argumentOf(call, i) and
     ap = mid.getAp() and
-    apa = ap.getApprox()
+    apa = ap.getApprox() and
+    config = mid.getConfiguration()
   )
 }
 
@@ -3557,12 +3585,14 @@ private predicate parameterCand(
 pragma[nomagic]
 private predicate pathIntoCallable0(
   PathNodeMid mid, DataFlowCallable callable, int i, CallContext outercc, DataFlowCall call,
-  AccessPath ap
+  AccessPath ap, Configuration config
 ) {
   exists(AccessPathApprox apa |
-    pathIntoArg(mid, i, outercc, call, ap, apa) and
+    pathIntoArg(mid, pragma[only_bind_into](i), outercc, call, ap, pragma[only_bind_into](apa),
+      pragma[only_bind_into](config)) and
     callable = resolveCall(call, outercc) and
-    parameterCand(callable, any(int j | j <= i and j >= i), apa, mid.getConfiguration())
+    parameterCand(callable, pragma[only_bind_into](i), pragma[only_bind_into](apa),
+      pragma[only_bind_into](config))
   )
 }
 
@@ -3571,12 +3601,13 @@ private predicate pathIntoCallable0(
  * before and after entering the callable are `outercc` and `innercc`,
  * respectively.
  */
+pragma[nomagic]
 private predicate pathIntoCallable(
   PathNodeMid mid, ParamNodeEx p, CallContext outercc, CallContextCall innercc, SummaryCtx sc,
-  DataFlowCall call
+  DataFlowCall call, Configuration config
 ) {
   exists(int i, DataFlowCallable callable, AccessPath ap |
-    pathIntoCallable0(mid, callable, i, outercc, call, ap) and
+    pathIntoCallable0(mid, callable, i, outercc, call, ap, config) and
     p.isParameterOf(callable, i) and
     (
       sc = TSummaryCtxSome(p, ap)
@@ -3606,18 +3637,23 @@ private predicate paramFlowsThrough(
     ap = mid.getAp() and
     apa = ap.getApprox() and
     pos = sc.getParameterPos() and
-    not kind.(ParamUpdateReturnKind).getPosition() = pos
+    // we don't expect a parameter to return stored in itself, unless explicitly allowed
+    (
+      not kind.(ParamUpdateReturnKind).getPosition() = pos
+      or
+      sc.getParamNode().allowParameterReturnInSelf()
+    )
   )
 }
 
 pragma[nomagic]
 private predicate pathThroughCallable0(
   DataFlowCall call, PathNodeMid mid, ReturnKindExt kind, CallContext cc, AccessPath ap,
-  AccessPathApprox apa
+  AccessPathApprox apa, Configuration config
 ) {
   exists(CallContext innercc, SummaryCtx sc |
-    pathIntoCallable(mid, _, cc, innercc, sc, call) and
-    paramFlowsThrough(kind, innercc, sc, ap, apa, unbindConf(mid.getConfiguration()))
+    pathIntoCallable(mid, _, cc, innercc, sc, call, config) and
+    paramFlowsThrough(kind, innercc, sc, ap, apa, config)
   )
 }
 
@@ -3627,9 +3663,9 @@ private predicate pathThroughCallable0(
  */
 pragma[noinline]
 private predicate pathThroughCallable(PathNodeMid mid, NodeEx out, CallContext cc, AccessPath ap) {
-  exists(DataFlowCall call, ReturnKindExt kind, AccessPathApprox apa |
-    pathThroughCallable0(call, mid, kind, cc, ap, apa) and
-    out = getAnOutNodeFlow(kind, call, apa, unbindConf(mid.getConfiguration()))
+  exists(DataFlowCall call, ReturnKindExt kind, AccessPathApprox apa, Configuration config |
+    pathThroughCallable0(call, mid, kind, cc, ap, apa, config) and
+    out = getAnOutNodeFlow(kind, call, apa, config)
   )
 }
 
@@ -3643,9 +3679,11 @@ private module Subpaths {
     PathNode arg, ParamNodeEx par, SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind,
     NodeEx out, AccessPath apout
   ) {
-    pathThroughCallable(arg, out, _, apout) and
-    pathIntoCallable(arg, par, _, innercc, sc, _) and
-    paramFlowsThrough(kind, innercc, sc, apout, _, unbindConf(arg.getConfiguration()))
+    exists(Configuration config |
+      pathThroughCallable(arg, out, _, pragma[only_bind_into](apout)) and
+      pathIntoCallable(arg, par, _, innercc, sc, _, config) and
+      paramFlowsThrough(kind, innercc, sc, pragma[only_bind_into](apout), _, unbindConf(config))
+    )
   }
 
   /**
@@ -4032,7 +4070,7 @@ private module FlowExploration {
      * The location spans column `startcolumn` of line `startline` to
      * column `endcolumn` of line `endline` in file `filepath`.
      * For more information, see
-     * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+     * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
      */
     predicate hasLocationInfo(
       string filepath, int startline, int startcolumn, int endline, int endcolumn

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll

@@ -110,12 +110,12 @@ abstract class Configuration extends string {
   /**
    * Holds if data may flow from some source to `sink` for this configuration.
    */
-  predicate hasFlowTo(Node sink) { hasFlow(_, sink) }
+  predicate hasFlowTo(Node sink) { this.hasFlow(_, sink) }
 
   /**
    * Holds if data may flow from some source to `sink` for this configuration.
    */
-  predicate hasFlowToExpr(DataFlowExpr sink) { hasFlowTo(exprNode(sink)) }
+  predicate hasFlowToExpr(DataFlowExpr sink) { this.hasFlowTo(exprNode(sink)) }
 
   /**
    * Gets the exploration limit for `hasPartialFlow` and `hasPartialFlowRev`
@@ -244,6 +244,8 @@ private class ParamNodeEx extends NodeEx {
   }
 
   int getPosition() { this.isParameterOf(_, result) }
+
+  predicate allowParameterReturnInSelf() { allowParameterReturnInSelfCached(this.asNode()) }
 }
 
 private class RetNodeEx extends NodeEx {
@@ -744,8 +746,12 @@ private module Stage1 {
       returnFlowCallableNodeCand(c, kind, config) and
       p.getEnclosingCallable() = c and
       exists(ap) and
-      // we don't expect a parameter to return stored in itself
-      not kind.(ParamUpdateReturnKind).getPosition() = p.getPosition()
+      // we don't expect a parameter to return stored in itself, unless explicitly allowed
+      (
+        not kind.(ParamUpdateReturnKind).getPosition() = p.getPosition()
+        or
+        p.allowParameterReturnInSelf()
+      )
     )
   }
 
@@ -1394,8 +1400,12 @@ private module Stage2 {
       fwdFlow(ret, any(CcCall ccc), apSome(ap), ap0, config) and
       kind = ret.getKind() and
       p.getPosition() = pos and
-      // we don't expect a parameter to return stored in itself
-      not kind.(ParamUpdateReturnKind).getPosition() = pos
+      // we don't expect a parameter to return stored in itself, unless explicitly allowed
+      (
+        not kind.(ParamUpdateReturnKind).getPosition() = pos
+        or
+        p.allowParameterReturnInSelf()
+      )
     )
   }
 
@@ -2083,8 +2093,12 @@ private module Stage3 {
       fwdFlow(ret, any(CcCall ccc), apSome(ap), ap0, config) and
       kind = ret.getKind() and
       p.getPosition() = pos and
-      // we don't expect a parameter to return stored in itself
-      not kind.(ParamUpdateReturnKind).getPosition() = pos
+      // we don't expect a parameter to return stored in itself, unless explicitly allowed
+      (
+        not kind.(ParamUpdateReturnKind).getPosition() = pos
+        or
+        p.allowParameterReturnInSelf()
+      )
     )
   }
 
@@ -2139,7 +2153,8 @@ private predicate expensiveLen2unfolding(TypedContent tc, Configuration config)
       ) and
     accessPathApproxCostLimits(apLimit, tupleLimit) and
     apLimit < tails and
-    tupleLimit < (tails - 1) * nodes
+    tupleLimit < (tails - 1) * nodes and
+    not tc.forceHighPrecision()
   )
 }
 
@@ -2842,8 +2857,12 @@ private module Stage4 {
       fwdFlow(ret, any(CcCall ccc), apSome(ap), ap0, config) and
       kind = ret.getKind() and
       p.getPosition() = pos and
-      // we don't expect a parameter to return stored in itself
-      not kind.(ParamUpdateReturnKind).getPosition() = pos
+      // we don't expect a parameter to return stored in itself, unless explicitly allowed
+      (
+        not kind.(ParamUpdateReturnKind).getPosition() = pos
+        or
+        p.allowParameterReturnInSelf()
+      )
     )
   }
 
@@ -2916,6 +2935,8 @@ private class SummaryCtxSome extends SummaryCtx, TSummaryCtxSome {
 
   int getParameterPos() { p.isParameterOf(_, result) }
 
+  ParamNodeEx getParamNode() { result = p }
+
   override string toString() { result = p + ": " + ap }
 
   predicate hasLocationInfo(
@@ -2973,12 +2994,15 @@ private AccessPathApprox getATail(AccessPathApprox apa, Configuration config) {
  * expected to be expensive. Holds with `unfold = true` otherwise.
  */
 private predicate evalUnfold(AccessPathApprox apa, boolean unfold, Configuration config) {
-  exists(int aps, int nodes, int apLimit, int tupleLimit |
-    aps = countPotentialAps(apa, config) and
-    nodes = countNodesUsingAccessPath(apa, config) and
-    accessPathCostLimits(apLimit, tupleLimit) and
-    if apLimit < aps and tupleLimit < (aps - 1) * nodes then unfold = false else unfold = true
-  )
+  if apa.getHead().forceHighPrecision()
+  then unfold = true
+  else
+    exists(int aps, int nodes, int apLimit, int tupleLimit |
+      aps = countPotentialAps(apa, config) and
+      nodes = countNodesUsingAccessPath(apa, config) and
+      accessPathCostLimits(apLimit, tupleLimit) and
+      if apLimit < aps and tupleLimit < (aps - 1) * nodes then unfold = false else unfold = true
+    )
 }
 
 /**
@@ -3166,7 +3190,7 @@ private class AccessPathCons extends AccessPath, TAccessPathCons {
   }
 
   override string toString() {
-    result = "[" + this.toStringImpl(true) + length().toString() + ")]"
+    result = "[" + this.toStringImpl(true) + this.length().toString() + ")]"
     or
     result = "[" + this.toStringImpl(false)
   }
@@ -3248,7 +3272,7 @@ class PathNode extends TPathNode {
    * The location spans column `startcolumn` of line `startline` to
    * column `endcolumn` of line `endline` in file `filepath`.
    * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn
@@ -3305,9 +3329,11 @@ abstract private class PathNodeImpl extends PathNode {
     result = " <" + this.(PathNodeMid).getCallContext().toString() + ">"
   }
 
-  override string toString() { result = this.getNodeEx().toString() + ppAp() }
+  override string toString() { result = this.getNodeEx().toString() + this.ppAp() }
 
-  override string toStringWithContext() { result = this.getNodeEx().toString() + ppAp() + ppCtx() }
+  override string toStringWithContext() {
+    result = this.getNodeEx().toString() + this.ppAp() + this.ppCtx()
+  }
 
   override predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn
@@ -3375,11 +3401,11 @@ private class PathNodeMid extends PathNodeImpl, TPathNodeMid {
 
   override PathNodeImpl getASuccessorImpl() {
     // an intermediate step to another intermediate node
-    result = getSuccMid()
+    result = this.getSuccMid()
     or
     // a final step to a sink via zero steps means we merge the last two steps to prevent trivial-looking edges
     exists(PathNodeMid mid, PathNodeSink sink |
-      mid = getSuccMid() and
+      mid = this.getSuccMid() and
       mid.getNodeEx() = sink.getNodeEx() and
       mid.getAp() instanceof AccessPathNil and
       sink.getConfiguration() = unbindConf(mid.getConfiguration()) and
@@ -3456,7 +3482,7 @@ private predicate pathStep(
   exists(TypedContent tc | pathReadStep(mid, node, ap.push(tc), tc, cc)) and
   sc = mid.getSummaryCtx()
   or
-  pathIntoCallable(mid, node, _, cc, sc, _) and ap = mid.getAp()
+  pathIntoCallable(mid, node, _, cc, sc, _, _) and ap = mid.getAp()
   or
   pathOutOfCallable(mid, node, cc) and ap = mid.getAp() and sc instanceof SummaryCtxNone
   or
@@ -3533,14 +3559,16 @@ private predicate pathOutOfCallable(PathNodeMid mid, NodeEx out, CallContext cc)
  */
 pragma[noinline]
 private predicate pathIntoArg(
-  PathNodeMid mid, int i, CallContext cc, DataFlowCall call, AccessPath ap, AccessPathApprox apa
+  PathNodeMid mid, int i, CallContext cc, DataFlowCall call, AccessPath ap, AccessPathApprox apa,
+  Configuration config
 ) {
   exists(ArgNode arg |
     arg = mid.getNodeEx().asNode() and
     cc = mid.getCallContext() and
     arg.argumentOf(call, i) and
     ap = mid.getAp() and
-    apa = ap.getApprox()
+    apa = ap.getApprox() and
+    config = mid.getConfiguration()
   )
 }
 
@@ -3557,12 +3585,14 @@ private predicate parameterCand(
 pragma[nomagic]
 private predicate pathIntoCallable0(
   PathNodeMid mid, DataFlowCallable callable, int i, CallContext outercc, DataFlowCall call,
-  AccessPath ap
+  AccessPath ap, Configuration config
 ) {
   exists(AccessPathApprox apa |
-    pathIntoArg(mid, i, outercc, call, ap, apa) and
+    pathIntoArg(mid, pragma[only_bind_into](i), outercc, call, ap, pragma[only_bind_into](apa),
+      pragma[only_bind_into](config)) and
     callable = resolveCall(call, outercc) and
-    parameterCand(callable, any(int j | j <= i and j >= i), apa, mid.getConfiguration())
+    parameterCand(callable, pragma[only_bind_into](i), pragma[only_bind_into](apa),
+      pragma[only_bind_into](config))
   )
 }
 
@@ -3571,12 +3601,13 @@ private predicate pathIntoCallable0(
  * before and after entering the callable are `outercc` and `innercc`,
  * respectively.
  */
+pragma[nomagic]
 private predicate pathIntoCallable(
   PathNodeMid mid, ParamNodeEx p, CallContext outercc, CallContextCall innercc, SummaryCtx sc,
-  DataFlowCall call
+  DataFlowCall call, Configuration config
 ) {
   exists(int i, DataFlowCallable callable, AccessPath ap |
-    pathIntoCallable0(mid, callable, i, outercc, call, ap) and
+    pathIntoCallable0(mid, callable, i, outercc, call, ap, config) and
     p.isParameterOf(callable, i) and
     (
       sc = TSummaryCtxSome(p, ap)
@@ -3606,18 +3637,23 @@ private predicate paramFlowsThrough(
     ap = mid.getAp() and
     apa = ap.getApprox() and
     pos = sc.getParameterPos() and
-    not kind.(ParamUpdateReturnKind).getPosition() = pos
+    // we don't expect a parameter to return stored in itself, unless explicitly allowed
+    (
+      not kind.(ParamUpdateReturnKind).getPosition() = pos
+      or
+      sc.getParamNode().allowParameterReturnInSelf()
+    )
   )
 }
 
 pragma[nomagic]
 private predicate pathThroughCallable0(
   DataFlowCall call, PathNodeMid mid, ReturnKindExt kind, CallContext cc, AccessPath ap,
-  AccessPathApprox apa
+  AccessPathApprox apa, Configuration config
 ) {
   exists(CallContext innercc, SummaryCtx sc |
-    pathIntoCallable(mid, _, cc, innercc, sc, call) and
-    paramFlowsThrough(kind, innercc, sc, ap, apa, unbindConf(mid.getConfiguration()))
+    pathIntoCallable(mid, _, cc, innercc, sc, call, config) and
+    paramFlowsThrough(kind, innercc, sc, ap, apa, config)
   )
 }
 
@@ -3627,9 +3663,9 @@ private predicate pathThroughCallable0(
  */
 pragma[noinline]
 private predicate pathThroughCallable(PathNodeMid mid, NodeEx out, CallContext cc, AccessPath ap) {
-  exists(DataFlowCall call, ReturnKindExt kind, AccessPathApprox apa |
-    pathThroughCallable0(call, mid, kind, cc, ap, apa) and
-    out = getAnOutNodeFlow(kind, call, apa, unbindConf(mid.getConfiguration()))
+  exists(DataFlowCall call, ReturnKindExt kind, AccessPathApprox apa, Configuration config |
+    pathThroughCallable0(call, mid, kind, cc, ap, apa, config) and
+    out = getAnOutNodeFlow(kind, call, apa, config)
   )
 }
 
@@ -3643,9 +3679,11 @@ private module Subpaths {
     PathNode arg, ParamNodeEx par, SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind,
     NodeEx out, AccessPath apout
   ) {
-    pathThroughCallable(arg, out, _, apout) and
-    pathIntoCallable(arg, par, _, innercc, sc, _) and
-    paramFlowsThrough(kind, innercc, sc, apout, _, unbindConf(arg.getConfiguration()))
+    exists(Configuration config |
+      pathThroughCallable(arg, out, _, pragma[only_bind_into](apout)) and
+      pathIntoCallable(arg, par, _, innercc, sc, _, config) and
+      paramFlowsThrough(kind, innercc, sc, pragma[only_bind_into](apout), _, unbindConf(config))
+    )
   }
 
   /**
@@ -4032,7 +4070,7 @@ private module FlowExploration {
      * The location spans column `startcolumn` of line `startline` to
      * column `endcolumn` of line `endline` in file `filepath`.
      * For more information, see
-     * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+     * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
      */
     predicate hasLocationInfo(
       string filepath, int startline, int startcolumn, int endline, int endcolumn

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll

@@ -110,12 +110,12 @@ abstract class Configuration extends string {
   /**
    * Holds if data may flow from some source to `sink` for this configuration.
    */
-  predicate hasFlowTo(Node sink) { hasFlow(_, sink) }
+  predicate hasFlowTo(Node sink) { this.hasFlow(_, sink) }
 
   /**
    * Holds if data may flow from some source to `sink` for this configuration.
    */
-  predicate hasFlowToExpr(DataFlowExpr sink) { hasFlowTo(exprNode(sink)) }
+  predicate hasFlowToExpr(DataFlowExpr sink) { this.hasFlowTo(exprNode(sink)) }
 
   /**
    * Gets the exploration limit for `hasPartialFlow` and `hasPartialFlowRev`
@@ -244,6 +244,8 @@ private class ParamNodeEx extends NodeEx {
   }
 
   int getPosition() { this.isParameterOf(_, result) }
+
+  predicate allowParameterReturnInSelf() { allowParameterReturnInSelfCached(this.asNode()) }
 }
 
 private class RetNodeEx extends NodeEx {
@@ -744,8 +746,12 @@ private module Stage1 {
       returnFlowCallableNodeCand(c, kind, config) and
       p.getEnclosingCallable() = c and
       exists(ap) and
-      // we don't expect a parameter to return stored in itself
-      not kind.(ParamUpdateReturnKind).getPosition() = p.getPosition()
+      // we don't expect a parameter to return stored in itself, unless explicitly allowed
+      (
+        not kind.(ParamUpdateReturnKind).getPosition() = p.getPosition()
+        or
+        p.allowParameterReturnInSelf()
+      )
     )
   }
 
@@ -1394,8 +1400,12 @@ private module Stage2 {
       fwdFlow(ret, any(CcCall ccc), apSome(ap), ap0, config) and
       kind = ret.getKind() and
       p.getPosition() = pos and
-      // we don't expect a parameter to return stored in itself
-      not kind.(ParamUpdateReturnKind).getPosition() = pos
+      // we don't expect a parameter to return stored in itself, unless explicitly allowed
+      (
+        not kind.(ParamUpdateReturnKind).getPosition() = pos
+        or
+        p.allowParameterReturnInSelf()
+      )
     )
   }
 
@@ -2083,8 +2093,12 @@ private module Stage3 {
       fwdFlow(ret, any(CcCall ccc), apSome(ap), ap0, config) and
       kind = ret.getKind() and
       p.getPosition() = pos and
-      // we don't expect a parameter to return stored in itself
-      not kind.(ParamUpdateReturnKind).getPosition() = pos
+      // we don't expect a parameter to return stored in itself, unless explicitly allowed
+      (
+        not kind.(ParamUpdateReturnKind).getPosition() = pos
+        or
+        p.allowParameterReturnInSelf()
+      )
     )
   }
 
@@ -2139,7 +2153,8 @@ private predicate expensiveLen2unfolding(TypedContent tc, Configuration config)
       ) and
     accessPathApproxCostLimits(apLimit, tupleLimit) and
     apLimit < tails and
-    tupleLimit < (tails - 1) * nodes
+    tupleLimit < (tails - 1) * nodes and
+    not tc.forceHighPrecision()
   )
 }
 
@@ -2842,8 +2857,12 @@ private module Stage4 {
       fwdFlow(ret, any(CcCall ccc), apSome(ap), ap0, config) and
       kind = ret.getKind() and
       p.getPosition() = pos and
-      // we don't expect a parameter to return stored in itself
-      not kind.(ParamUpdateReturnKind).getPosition() = pos
+      // we don't expect a parameter to return stored in itself, unless explicitly allowed
+      (
+        not kind.(ParamUpdateReturnKind).getPosition() = pos
+        or
+        p.allowParameterReturnInSelf()
+      )
     )
   }
 
@@ -2916,6 +2935,8 @@ private class SummaryCtxSome extends SummaryCtx, TSummaryCtxSome {
 
   int getParameterPos() { p.isParameterOf(_, result) }
 
+  ParamNodeEx getParamNode() { result = p }
+
   override string toString() { result = p + ": " + ap }
 
   predicate hasLocationInfo(
@@ -2973,12 +2994,15 @@ private AccessPathApprox getATail(AccessPathApprox apa, Configuration config) {
  * expected to be expensive. Holds with `unfold = true` otherwise.
  */
 private predicate evalUnfold(AccessPathApprox apa, boolean unfold, Configuration config) {
-  exists(int aps, int nodes, int apLimit, int tupleLimit |
-    aps = countPotentialAps(apa, config) and
-    nodes = countNodesUsingAccessPath(apa, config) and
-    accessPathCostLimits(apLimit, tupleLimit) and
-    if apLimit < aps and tupleLimit < (aps - 1) * nodes then unfold = false else unfold = true
-  )
+  if apa.getHead().forceHighPrecision()
+  then unfold = true
+  else
+    exists(int aps, int nodes, int apLimit, int tupleLimit |
+      aps = countPotentialAps(apa, config) and
+      nodes = countNodesUsingAccessPath(apa, config) and
+      accessPathCostLimits(apLimit, tupleLimit) and
+      if apLimit < aps and tupleLimit < (aps - 1) * nodes then unfold = false else unfold = true
+    )
 }
 
 /**
@@ -3166,7 +3190,7 @@ private class AccessPathCons extends AccessPath, TAccessPathCons {
   }
 
   override string toString() {
-    result = "[" + this.toStringImpl(true) + length().toString() + ")]"
+    result = "[" + this.toStringImpl(true) + this.length().toString() + ")]"
     or
     result = "[" + this.toStringImpl(false)
   }
@@ -3248,7 +3272,7 @@ class PathNode extends TPathNode {
    * The location spans column `startcolumn` of line `startline` to
    * column `endcolumn` of line `endline` in file `filepath`.
    * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn
@@ -3305,9 +3329,11 @@ abstract private class PathNodeImpl extends PathNode {
     result = " <" + this.(PathNodeMid).getCallContext().toString() + ">"
   }
 
-  override string toString() { result = this.getNodeEx().toString() + ppAp() }
+  override string toString() { result = this.getNodeEx().toString() + this.ppAp() }
 
-  override string toStringWithContext() { result = this.getNodeEx().toString() + ppAp() + ppCtx() }
+  override string toStringWithContext() {
+    result = this.getNodeEx().toString() + this.ppAp() + this.ppCtx()
+  }
 
   override predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn
@@ -3375,11 +3401,11 @@ private class PathNodeMid extends PathNodeImpl, TPathNodeMid {
 
   override PathNodeImpl getASuccessorImpl() {
     // an intermediate step to another intermediate node
-    result = getSuccMid()
+    result = this.getSuccMid()
     or
     // a final step to a sink via zero steps means we merge the last two steps to prevent trivial-looking edges
     exists(PathNodeMid mid, PathNodeSink sink |
-      mid = getSuccMid() and
+      mid = this.getSuccMid() and
       mid.getNodeEx() = sink.getNodeEx() and
       mid.getAp() instanceof AccessPathNil and
       sink.getConfiguration() = unbindConf(mid.getConfiguration()) and
@@ -3456,7 +3482,7 @@ private predicate pathStep(
   exists(TypedContent tc | pathReadStep(mid, node, ap.push(tc), tc, cc)) and
   sc = mid.getSummaryCtx()
   or
-  pathIntoCallable(mid, node, _, cc, sc, _) and ap = mid.getAp()
+  pathIntoCallable(mid, node, _, cc, sc, _, _) and ap = mid.getAp()
   or
   pathOutOfCallable(mid, node, cc) and ap = mid.getAp() and sc instanceof SummaryCtxNone
   or
@@ -3533,14 +3559,16 @@ private predicate pathOutOfCallable(PathNodeMid mid, NodeEx out, CallContext cc)
  */
 pragma[noinline]
 private predicate pathIntoArg(
-  PathNodeMid mid, int i, CallContext cc, DataFlowCall call, AccessPath ap, AccessPathApprox apa
+  PathNodeMid mid, int i, CallContext cc, DataFlowCall call, AccessPath ap, AccessPathApprox apa,
+  Configuration config
 ) {
   exists(ArgNode arg |
     arg = mid.getNodeEx().asNode() and
     cc = mid.getCallContext() and
     arg.argumentOf(call, i) and
     ap = mid.getAp() and
-    apa = ap.getApprox()
+    apa = ap.getApprox() and
+    config = mid.getConfiguration()
   )
 }
 
@@ -3557,12 +3585,14 @@ private predicate parameterCand(
 pragma[nomagic]
 private predicate pathIntoCallable0(
   PathNodeMid mid, DataFlowCallable callable, int i, CallContext outercc, DataFlowCall call,
-  AccessPath ap
+  AccessPath ap, Configuration config
 ) {
   exists(AccessPathApprox apa |
-    pathIntoArg(mid, i, outercc, call, ap, apa) and
+    pathIntoArg(mid, pragma[only_bind_into](i), outercc, call, ap, pragma[only_bind_into](apa),
+      pragma[only_bind_into](config)) and
     callable = resolveCall(call, outercc) and
-    parameterCand(callable, any(int j | j <= i and j >= i), apa, mid.getConfiguration())
+    parameterCand(callable, pragma[only_bind_into](i), pragma[only_bind_into](apa),
+      pragma[only_bind_into](config))
   )
 }
 
@@ -3571,12 +3601,13 @@ private predicate pathIntoCallable0(
  * before and after entering the callable are `outercc` and `innercc`,
  * respectively.
  */
+pragma[nomagic]
 private predicate pathIntoCallable(
   PathNodeMid mid, ParamNodeEx p, CallContext outercc, CallContextCall innercc, SummaryCtx sc,
-  DataFlowCall call
+  DataFlowCall call, Configuration config
 ) {
   exists(int i, DataFlowCallable callable, AccessPath ap |
-    pathIntoCallable0(mid, callable, i, outercc, call, ap) and
+    pathIntoCallable0(mid, callable, i, outercc, call, ap, config) and
     p.isParameterOf(callable, i) and
     (
       sc = TSummaryCtxSome(p, ap)
@@ -3606,18 +3637,23 @@ private predicate paramFlowsThrough(
     ap = mid.getAp() and
     apa = ap.getApprox() and
     pos = sc.getParameterPos() and
-    not kind.(ParamUpdateReturnKind).getPosition() = pos
+    // we don't expect a parameter to return stored in itself, unless explicitly allowed
+    (
+      not kind.(ParamUpdateReturnKind).getPosition() = pos
+      or
+      sc.getParamNode().allowParameterReturnInSelf()
+    )
   )
 }
 
 pragma[nomagic]
 private predicate pathThroughCallable0(
   DataFlowCall call, PathNodeMid mid, ReturnKindExt kind, CallContext cc, AccessPath ap,
-  AccessPathApprox apa
+  AccessPathApprox apa, Configuration config
 ) {
   exists(CallContext innercc, SummaryCtx sc |
-    pathIntoCallable(mid, _, cc, innercc, sc, call) and
-    paramFlowsThrough(kind, innercc, sc, ap, apa, unbindConf(mid.getConfiguration()))
+    pathIntoCallable(mid, _, cc, innercc, sc, call, config) and
+    paramFlowsThrough(kind, innercc, sc, ap, apa, config)
   )
 }
 
@@ -3627,9 +3663,9 @@ private predicate pathThroughCallable0(
  */
 pragma[noinline]
 private predicate pathThroughCallable(PathNodeMid mid, NodeEx out, CallContext cc, AccessPath ap) {
-  exists(DataFlowCall call, ReturnKindExt kind, AccessPathApprox apa |
-    pathThroughCallable0(call, mid, kind, cc, ap, apa) and
-    out = getAnOutNodeFlow(kind, call, apa, unbindConf(mid.getConfiguration()))
+  exists(DataFlowCall call, ReturnKindExt kind, AccessPathApprox apa, Configuration config |
+    pathThroughCallable0(call, mid, kind, cc, ap, apa, config) and
+    out = getAnOutNodeFlow(kind, call, apa, config)
   )
 }
 
@@ -3643,9 +3679,11 @@ private module Subpaths {
     PathNode arg, ParamNodeEx par, SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind,
     NodeEx out, AccessPath apout
   ) {
-    pathThroughCallable(arg, out, _, apout) and
-    pathIntoCallable(arg, par, _, innercc, sc, _) and
-    paramFlowsThrough(kind, innercc, sc, apout, _, unbindConf(arg.getConfiguration()))
+    exists(Configuration config |
+      pathThroughCallable(arg, out, _, pragma[only_bind_into](apout)) and
+      pathIntoCallable(arg, par, _, innercc, sc, _, config) and
+      paramFlowsThrough(kind, innercc, sc, pragma[only_bind_into](apout), _, unbindConf(config))
+    )
   }
 
   /**
@@ -4032,7 +4070,7 @@ private module FlowExploration {
      * The location spans column `startcolumn` of line `startline` to
      * column `endcolumn` of line `endline` in file `filepath`.
      * For more information, see
-     * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+     * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
      */
     predicate hasLocationInfo(
       string filepath, int startline, int startcolumn, int endline, int endcolumn

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplCommon.qll

@@ -801,6 +801,9 @@ private module Cached {
     exists(Node n | getNodeEnclosingCallable(n) = callable | isUnreachableInCallCached(n, call))
   }
 
+  cached
+  predicate allowParameterReturnInSelfCached(ParamNode p) { allowParameterReturnInSelf(p) }
+
   cached
   newtype TCallContext =
     TAnyCallContext() or
@@ -937,7 +940,7 @@ class CallContextSpecificCall extends CallContextCall, TSpecificCall {
   }
 
   override predicate relevantFor(DataFlowCallable callable) {
-    recordDataFlowCallSite(getCall(), callable)
+    recordDataFlowCallSite(this.getCall(), callable)
   }
 
   override predicate matchesCall(DataFlowCall call) { call = this.getCall() }
@@ -1236,6 +1239,13 @@ class TypedContent extends MkTypedContent {
 
   /** Gets a textual representation of this content. */
   string toString() { result = c.toString() }
+
+  /**
+   * Holds if access paths with this `TypedContent` at their head always should
+   * be tracked at high precision. This disables adaptive access path precision
+   * for such access paths.
+   */
+  predicate forceHighPrecision() { forceHighPrecision(c) }
 }
 
 /**
@@ -1250,7 +1260,7 @@ abstract class AccessPathFront extends TAccessPathFront {
 
   TypedContent getHead() { this = TFrontHead(result) }
 
-  predicate isClearedAt(Node n) { clearsContentCached(n, getHead().getContent()) }
+  predicate isClearedAt(Node n) { clearsContentCached(n, this.getHead().getContent()) }
 }
 
 class AccessPathFrontNil extends AccessPathFront, TFrontNil {

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplConsistency.qll

@@ -175,6 +175,7 @@ module Consistency {
 
   query predicate postWithInFlow(Node n, string msg) {
     isPostUpdateNode(n) and
+    not clearsContent(n, _) and
     simpleLocalFlowStep(_, n) and
     msg = "PostUpdateNode should not be the target of local flow."
   }

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll

@@ -152,6 +152,7 @@ class DataFlowExpr = Expr;
  * Flow comes from definitions, uses and refinements.
  */
 // TODO: Consider constraining `nodeFrom` and `nodeTo` to be in the same scope.
+// If they have different enclosing callables, we get consistency errors.
 module EssaFlow {
   predicate essaFlowStep(Node nodeFrom, Node nodeTo) {
     // Definition
@@ -200,6 +201,9 @@ module EssaFlow {
     // If expressions
     nodeFrom.asCfgNode() = nodeTo.asCfgNode().(IfExprNode).getAnOperand()
     or
+    // boolean inline expressions such as `x or y` or `x and y`
+    nodeFrom.asCfgNode() = nodeTo.asCfgNode().(BoolExprNode).getAnOperand()
+    or
     // Flow inside an unpacking assignment
     iterableUnpackingFlowStep(nodeFrom, nodeTo)
     or
@@ -225,35 +229,60 @@ module EssaFlow {
 //--------
 /**
  * This is the local flow predicate that is used as a building block in global
- * data flow. It is a strict subset of the `localFlowStep` predicate, as it
- * excludes SSA flow through instance fields.
+ * data flow.
+ *
+ * Local flow can happen either at import time, when the module is initialised
+ * or at runtime when callables in the module are called.
  */
 predicate simpleLocalFlowStep(Node nodeFrom, Node nodeTo) {
-  // If there is ESSA-flow out of a node `node`, we want flow
+  // If there is local flow out of a node `node`, we want flow
   // both out of `node` and any post-update node of `node`.
   exists(Node node |
-    EssaFlow::essaFlowStep(node, nodeTo) and
     nodeFrom = update(node) and
     (
-      not node instanceof EssaNode or
-      not nodeTo instanceof EssaNode or
-      localEssaStep(node, nodeTo)
+      importTimeLocalFlowStep(node, nodeTo) or
+      runtimeLocalFlowStep(node, nodeTo)
     )
   )
 }
 
 /**
- * Holds if there is an Essa flow step from `nodeFrom` to `nodeTo` that does not switch between
- * local and global SSA variables.
+ * Holds if `node` is found at the top level of a module.
  */
-private predicate localEssaStep(EssaNode nodeFrom, EssaNode nodeTo) {
-  EssaFlow::essaFlowStep(nodeFrom, nodeTo) and
-  (
-    nodeFrom.getVar() instanceof GlobalSsaVariable and
-    nodeTo.getVar() instanceof GlobalSsaVariable
-    or
-    not nodeFrom.getVar() instanceof GlobalSsaVariable and
-    not nodeTo.getVar() instanceof GlobalSsaVariable
+pragma[inline]
+predicate isTopLevel(Node node) { node.getScope() instanceof Module }
+
+/** Holds if there is local flow from `nodeFrom` to `nodeTo` at import time. */
+predicate importTimeLocalFlowStep(Node nodeFrom, Node nodeTo) {
+  // As a proxy for whether statements can be executed at import time,
+  // we check if they appear at the top level.
+  // This will miss statements inside functions called from the top level.
+  isTopLevel(nodeFrom) and
+  isTopLevel(nodeTo) and
+  EssaFlow::essaFlowStep(nodeFrom, nodeTo)
+}
+
+/** Holds if there is local flow from `nodeFrom` to `nodeTo` at runtime. */
+predicate runtimeLocalFlowStep(Node nodeFrom, Node nodeTo) {
+  // Anything not at the top level can be executed at runtime.
+  not isTopLevel(nodeFrom) and
+  not isTopLevel(nodeTo) and
+  EssaFlow::essaFlowStep(nodeFrom, nodeTo)
+}
+
+/** `ModuleVariable`s are accessed via jump steps at runtime. */
+predicate runtimeJumpStep(Node nodeFrom, Node nodeTo) {
+  // Module variable read
+  nodeFrom.(ModuleVariableNode).getARead() = nodeTo
+  or
+  // Module variable write
+  nodeFrom = nodeTo.(ModuleVariableNode).getAWrite()
+  or
+  // Setting the possible values of the variable at the end of import time
+  exists(SsaVariable def |
+    def = any(SsaVariable var).getAnUltimateDefinition() and
+    def.getDefinition() = nodeFrom.asCfgNode() and
+    def.getVariable() = nodeTo.(ModuleVariableNode).getVariable()
   )
 }
 
@@ -581,11 +610,11 @@ class DataFlowLambda extends DataFlowCallable, TLambda {
 
   override string toString() { result = lambda.toString() }
 
-  override CallNode getACall() { result = getCallableValue().getACall() }
+  override CallNode getACall() { result = this.getCallableValue().getACall() }
 
   override Scope getScope() { result = lambda.getEvaluatingScope() }
 
-  override NameNode getParameter(int n) { result = getParameter(getCallableValue(), n) }
+  override NameNode getParameter(int n) { result = getParameter(this.getCallableValue(), n) }
 
   override string getName() { result = "Lambda callable" }
 
@@ -857,11 +886,7 @@ string ppReprType(DataFlowType t) { none() }
  * taken into account.
  */
 predicate jumpStep(Node nodeFrom, Node nodeTo) {
-  // Module variable read
-  nodeFrom.(ModuleVariableNode).getARead() = nodeTo
-  or
-  // Module variable write
-  nodeFrom = nodeTo.(ModuleVariableNode).getAWrite()
+  runtimeJumpStep(nodeFrom, nodeTo)
   or
   // Read of module attribute:
   exists(AttrRead r, ModuleValue mv |
@@ -1620,6 +1645,12 @@ predicate isImmutableOrUnobservable(Node n) { none() }
 
 int accessPathLimit() { result = 5 }
 
+/**
+ * Holds if access paths with `c` at their head always should be tracked at high
+ * precision. This disables adaptive access path precision for such access paths.
+ */
+predicate forceHighPrecision(Content c) { none() }
+
 /** Holds if `n` should be hidden from path explanations. */
 predicate nodeIsHidden(Node n) { none() }
 
@@ -1633,3 +1664,12 @@ predicate lambdaCall(DataFlowCall call, LambdaCallKind kind, Node receiver) { no
 
 /** Extra data-flow steps needed for lambda flow analysis. */
 predicate additionalLambdaFlowStep(Node nodeFrom, Node nodeTo, boolean preservesValue) { none() }
+
+/**
+ * Holds if flow is allowed to pass from parameter `p` and back to itself as a
+ * side-effect, resulting in a summary from `p` to itself.
+ *
+ * One example would be to allow flow like `p.foo = p.bar;`, which is disallowed
+ * by default as a heuristic.
+ */
+predicate allowParameterReturnInSelf(ParameterNode p) { none() }

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll

@@ -102,7 +102,7 @@ class Node extends TNode {
    * The location spans column `startcolumn` of line `startline` to
    * column `endcolumn` of line `endline` in file `filepath`.
    * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn
@@ -332,7 +332,7 @@ class ModuleVariableNode extends Node, TModuleVariableNode {
   override Scope getScope() { result = mod }
 
   override string toString() {
-    result = "ModuleVariableNode for " + var.toString() + " in " + mod.toString()
+    result = "ModuleVariableNode for " + mod.getName() + "." + var.getId()
   }
 
   /** Gets the module in which this variable appears. */

python/ql/lib/semmle/python/dataflow/new/internal/LocalSources.qll

@@ -62,12 +62,12 @@ class LocalSourceNode extends Node {
   /**
    * Gets a read of attribute `attrName` on this node.
    */
-  AttrRead getAnAttributeRead(string attrName) { result = getAnAttributeReference(attrName) }
+  AttrRead getAnAttributeRead(string attrName) { result = this.getAnAttributeReference(attrName) }
 
   /**
    * Gets a write of attribute `attrName` on this node.
    */
-  AttrWrite getAnAttributeWrite(string attrName) { result = getAnAttributeReference(attrName) }
+  AttrWrite getAnAttributeWrite(string attrName) { result = this.getAnAttributeReference(attrName) }
 
   /**
    * Gets a reference (read or write) of any attribute on this node.
@@ -81,12 +81,12 @@ class LocalSourceNode extends Node {
   /**
    * Gets a read of any attribute on this node.
    */
-  AttrRead getAnAttributeRead() { result = getAnAttributeReference() }
+  AttrRead getAnAttributeRead() { result = this.getAnAttributeReference() }
 
   /**
    * Gets a write of any attribute on this node.
    */
-  AttrWrite getAnAttributeWrite() { result = getAnAttributeReference() }
+  AttrWrite getAnAttributeWrite() { result = this.getAnAttributeReference() }
 
   /**
    * Gets a call to this node.

python/ql/lib/semmle/python/dataflow/new/internal/PrintNode.qll

@@ -58,7 +58,6 @@ string prettyNode(DataFlow::Node node) {
  */
 bindingset[node]
 string prettyNodeForInlineTest(DataFlow::Node node) {
-  exists(node.asExpr()) and
   result = prettyExpr(node.asExpr())
   or
   exists(Expr e | e = node.(DataFlow::PostUpdateNode).getPreUpdateNode().asExpr() |

python/ql/lib/semmle/python/dataflow/new/internal/tainttracking1/TaintTrackingImpl.qll

@@ -75,24 +75,26 @@ abstract class Configuration extends DataFlow::Configuration {
   predicate isSanitizer(DataFlow::Node node) { none() }
 
   final override predicate isBarrier(DataFlow::Node node) {
-    isSanitizer(node) or
+    this.isSanitizer(node) or
     defaultTaintSanitizer(node)
   }
 
   /** Holds if taint propagation into `node` is prohibited. */
   predicate isSanitizerIn(DataFlow::Node node) { none() }
 
-  final override predicate isBarrierIn(DataFlow::Node node) { isSanitizerIn(node) }
+  final override predicate isBarrierIn(DataFlow::Node node) { this.isSanitizerIn(node) }
 
   /** Holds if taint propagation out of `node` is prohibited. */
   predicate isSanitizerOut(DataFlow::Node node) { none() }
 
-  final override predicate isBarrierOut(DataFlow::Node node) { isSanitizerOut(node) }
+  final override predicate isBarrierOut(DataFlow::Node node) { this.isSanitizerOut(node) }
 
   /** Holds if taint propagation through nodes guarded by `guard` is prohibited. */
   predicate isSanitizerGuard(DataFlow::BarrierGuard guard) { none() }
 
-  final override predicate isBarrierGuard(DataFlow::BarrierGuard guard) { isSanitizerGuard(guard) }
+  final override predicate isBarrierGuard(DataFlow::BarrierGuard guard) {
+    this.isSanitizerGuard(guard)
+  }
 
   /**
    * Holds if the additional taint propagation step from `node1` to `node2`
@@ -101,7 +103,7 @@ abstract class Configuration extends DataFlow::Configuration {
   predicate isAdditionalTaintStep(DataFlow::Node node1, DataFlow::Node node2) { none() }
 
   final override predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
-    isAdditionalTaintStep(node1, node2) or
+    this.isAdditionalTaintStep(node1, node2) or
     defaultAdditionalTaintStep(node1, node2)
   }
 

python/ql/lib/semmle/python/dataflow/new/internal/tainttracking2/TaintTrackingImpl.qll

@@ -75,24 +75,26 @@ abstract class Configuration extends DataFlow::Configuration {
   predicate isSanitizer(DataFlow::Node node) { none() }
 
   final override predicate isBarrier(DataFlow::Node node) {
-    isSanitizer(node) or
+    this.isSanitizer(node) or
     defaultTaintSanitizer(node)
   }
 
   /** Holds if taint propagation into `node` is prohibited. */
   predicate isSanitizerIn(DataFlow::Node node) { none() }
 
-  final override predicate isBarrierIn(DataFlow::Node node) { isSanitizerIn(node) }
+  final override predicate isBarrierIn(DataFlow::Node node) { this.isSanitizerIn(node) }
 
   /** Holds if taint propagation out of `node` is prohibited. */
   predicate isSanitizerOut(DataFlow::Node node) { none() }
 
-  final override predicate isBarrierOut(DataFlow::Node node) { isSanitizerOut(node) }
+  final override predicate isBarrierOut(DataFlow::Node node) { this.isSanitizerOut(node) }
 
   /** Holds if taint propagation through nodes guarded by `guard` is prohibited. */
   predicate isSanitizerGuard(DataFlow::BarrierGuard guard) { none() }
 
-  final override predicate isBarrierGuard(DataFlow::BarrierGuard guard) { isSanitizerGuard(guard) }
+  final override predicate isBarrierGuard(DataFlow::BarrierGuard guard) {
+    this.isSanitizerGuard(guard)
+  }
 
   /**
    * Holds if the additional taint propagation step from `node1` to `node2`
@@ -101,7 +103,7 @@ abstract class Configuration extends DataFlow::Configuration {
   predicate isAdditionalTaintStep(DataFlow::Node node1, DataFlow::Node node2) { none() }
 
   final override predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
-    isAdditionalTaintStep(node1, node2) or
+    this.isAdditionalTaintStep(node1, node2) or
     defaultAdditionalTaintStep(node1, node2)
   }
 

python/ql/lib/semmle/python/dataflow/new/internal/tainttracking3/TaintTrackingImpl.qll

@@ -75,24 +75,26 @@ abstract class Configuration extends DataFlow::Configuration {
   predicate isSanitizer(DataFlow::Node node) { none() }
 
   final override predicate isBarrier(DataFlow::Node node) {
-    isSanitizer(node) or
+    this.isSanitizer(node) or
     defaultTaintSanitizer(node)
   }
 
   /** Holds if taint propagation into `node` is prohibited. */
   predicate isSanitizerIn(DataFlow::Node node) { none() }
 
-  final override predicate isBarrierIn(DataFlow::Node node) { isSanitizerIn(node) }
+  final override predicate isBarrierIn(DataFlow::Node node) { this.isSanitizerIn(node) }
 
   /** Holds if taint propagation out of `node` is prohibited. */
   predicate isSanitizerOut(DataFlow::Node node) { none() }
 
-  final override predicate isBarrierOut(DataFlow::Node node) { isSanitizerOut(node) }
+  final override predicate isBarrierOut(DataFlow::Node node) { this.isSanitizerOut(node) }
 
   /** Holds if taint propagation through nodes guarded by `guard` is prohibited. */
   predicate isSanitizerGuard(DataFlow::BarrierGuard guard) { none() }
 
-  final override predicate isBarrierGuard(DataFlow::BarrierGuard guard) { isSanitizerGuard(guard) }
+  final override predicate isBarrierGuard(DataFlow::BarrierGuard guard) {
+    this.isSanitizerGuard(guard)
+  }
 
   /**
    * Holds if the additional taint propagation step from `node1` to `node2`
@@ -101,7 +103,7 @@ abstract class Configuration extends DataFlow::Configuration {
   predicate isAdditionalTaintStep(DataFlow::Node node1, DataFlow::Node node2) { none() }
 
   final override predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
-    isAdditionalTaintStep(node1, node2) or
+    this.isAdditionalTaintStep(node1, node2) or
     defaultAdditionalTaintStep(node1, node2)
   }
 

python/ql/lib/semmle/python/dataflow/new/internal/tainttracking4/TaintTrackingImpl.qll

@@ -75,24 +75,26 @@ abstract class Configuration extends DataFlow::Configuration {
   predicate isSanitizer(DataFlow::Node node) { none() }
 
   final override predicate isBarrier(DataFlow::Node node) {
-    isSanitizer(node) or
+    this.isSanitizer(node) or
     defaultTaintSanitizer(node)
   }
 
   /** Holds if taint propagation into `node` is prohibited. */
   predicate isSanitizerIn(DataFlow::Node node) { none() }
 
-  final override predicate isBarrierIn(DataFlow::Node node) { isSanitizerIn(node) }
+  final override predicate isBarrierIn(DataFlow::Node node) { this.isSanitizerIn(node) }
 
   /** Holds if taint propagation out of `node` is prohibited. */
   predicate isSanitizerOut(DataFlow::Node node) { none() }
 
-  final override predicate isBarrierOut(DataFlow::Node node) { isSanitizerOut(node) }
+  final override predicate isBarrierOut(DataFlow::Node node) { this.isSanitizerOut(node) }
 
   /** Holds if taint propagation through nodes guarded by `guard` is prohibited. */
   predicate isSanitizerGuard(DataFlow::BarrierGuard guard) { none() }
 
-  final override predicate isBarrierGuard(DataFlow::BarrierGuard guard) { isSanitizerGuard(guard) }
+  final override predicate isBarrierGuard(DataFlow::BarrierGuard guard) {
+    this.isSanitizerGuard(guard)
+  }
 
   /**
    * Holds if the additional taint propagation step from `node1` to `node2`
@@ -101,7 +103,7 @@ abstract class Configuration extends DataFlow::Configuration {
   predicate isAdditionalTaintStep(DataFlow::Node node1, DataFlow::Node node2) { none() }
 
   final override predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
-    isAdditionalTaintStep(node1, node2) or
+    this.isAdditionalTaintStep(node1, node2) or
     defaultAdditionalTaintStep(node1, node2)
   }
 

python/ql/lib/semmle/python/dataflow/old/TaintTracking.qll

@@ -384,7 +384,7 @@ abstract class TaintSource extends @py_flow_node {
    * The location spans column `startcolumn` of line `startline` to
    * column `endcolumn` of line `endline` in file `filepath`.
    * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn
@@ -498,7 +498,7 @@ abstract class TaintSink extends @py_flow_node {
    * The location spans column `startcolumn` of line `startline` to
    * column `endcolumn` of line `endline` in file `filepath`.
    * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn

python/ql/lib/semmle/python/essa/Definitions.qll

@@ -225,9 +225,9 @@ class ModuleVariable extends SsaSourceVariable {
   }
 
   override ControlFlowNode getAnImplicitUse() {
-    result = global_variable_callnode()
+    result = this.global_variable_callnode()
     or
-    result = global_variable_import()
+    result = this.global_variable_import()
     or
     exists(ImportTimeScope scope | scope.entryEdge(result, _) |
       this = scope.getOuterVariable(_) or

python/ql/lib/semmle/python/essa/Essa.qll

@@ -41,7 +41,7 @@ class EssaVariable extends TEssaDefinition {
    */
   ControlFlowNode getASourceUse() {
     exists(SsaSourceVariable var |
-      result = use_for_var(var) and
+      result = this.use_for_var(var) and
       result = var.getASourceUse()
     )
   }
@@ -258,7 +258,7 @@ class PhiFunction extends EssaDefinition, TPhiFunction {
   /** Gets another definition of the same source variable that reaches this definition. */
   private EssaDefinition reachingDefinition(BasicBlock pred) {
     result.getScope() = this.getScope() and
-    result.getSourceVariable() = pred_var(pred) and
+    result.getSourceVariable() = this.pred_var(pred) and
     result.reachesEndOfBlock(pred)
   }
 

python/ql/lib/semmle/python/frameworks/Aiohttp.qll

@@ -424,7 +424,7 @@ module AiohttpWebModel {
 
       override string getAttributeName() { none() }
 
-      override string getMethodName() { result in ["read_nowait"] }
+      override string getMethodName() { result = "read_nowait" }
 
       override string getAsyncMethodName() {
         result in [

python/ql/lib/semmle/python/frameworks/Cryptodome.qll

@@ -116,7 +116,7 @@ private module CryptodomeModel {
         ] and
       this =
         API::moduleImport(["Crypto", "Cryptodome"])
-            .getMember(["Cipher"])
+            .getMember("Cipher")
             .getMember(cipherName)
             .getMember("new")
             .getReturn()
@@ -135,21 +135,21 @@ private module CryptodomeModel {
       or
       // for the following methods, method signatures can be found in
       // https://pycryptodome.readthedocs.io/en/latest/src/cipher/modern.html
-      methodName in ["update"] and
+      methodName = "update" and
       result in [this.getArg(0), this.getArgByName("data")]
       or
       // although `mac_tag` is used as the parameter name in the spec above, some implementations use `received_mac_tag`, for an example, see
       // https://github.com/Legrandin/pycryptodome/blob/5dace638b70ac35bb5d9b565f3e75f7869c9d851/lib/Crypto/Cipher/ChaCha20_Poly1305.py#L207
-      methodName in ["verify"] and
+      methodName = "verify" and
       result in [this.getArg(0), this.getArgByName(["mac_tag", "received_mac_tag"])]
       or
-      methodName in ["hexverify"] and
+      methodName = "hexverify" and
       result in [this.getArg(0), this.getArgByName("mac_tag_hex")]
       or
-      methodName in ["encrypt_and_digest"] and
+      methodName = "encrypt_and_digest" and
       result in [this.getArg(0), this.getArgByName("plaintext")]
       or
-      methodName in ["decrypt_and_verify"] and
+      methodName = "decrypt_and_verify" and
       result in [
           this.getArg(0), this.getArgByName("ciphertext"), this.getArg(1),
           this.getArgByName("mac_tag")
@@ -169,7 +169,7 @@ private module CryptodomeModel {
       methodName in ["sign", "verify"] and
       this =
         API::moduleImport(["Crypto", "Cryptodome"])
-            .getMember(["Signature"])
+            .getMember("Signature")
             .getMember(signatureName)
             .getMember("new")
             .getReturn()
@@ -185,11 +185,11 @@ private module CryptodomeModel {
       methodName = "sign" and
       result in [this.getArg(0), this.getArgByName("msg_hash")] // Cryptodome.Hash instance
       or
-      methodName in ["verify"] and
+      methodName = "verify" and
       (
-        result in [this.getArg(0), this.getArgByName(["msg_hash"])] // Cryptodome.Hash instance
+        result in [this.getArg(0), this.getArgByName("msg_hash")] // Cryptodome.Hash instance
         or
-        result in [this.getArg(1), this.getArgByName(["signature"])]
+        result in [this.getArg(1), this.getArgByName("signature")]
       )
     }
   }
@@ -204,7 +204,7 @@ private module CryptodomeModel {
     CryptodomeGenericHashOperation() {
       exists(API::Node hashModule |
         hashModule =
-          API::moduleImport(["Crypto", "Cryptodome"]).getMember(["Hash"]).getMember(hashName)
+          API::moduleImport(["Crypto", "Cryptodome"]).getMember("Hash").getMember(hashName)
       |
         this = hashModule.getMember("new").getACall()
         or

python/ql/lib/semmle/python/frameworks/Dill.qll

@@ -1,5 +1,5 @@
 /**
- * Provides classes modeling security-relevant aspects of the 'dill' package.
+ * Provides classes modeling security-relevant aspects of the `dill` PyPI package.
  * See https://pypi.org/project/dill/.
  */
 
@@ -10,18 +10,41 @@ private import semmle.python.Concepts
 private import semmle.python.ApiGraphs
 
 /**
- * A call to `dill.loads`
- * See https://pypi.org/project/dill/ (which currently refers you
- * to https://docs.python.org/3/library/pickle.html#pickle.loads)
+ * Provides models for the `dill` PyPI package.
+ * See https://pypi.org/project/dill/.
  */
-private class DillLoadsCall extends Decoding::Range, DataFlow::CallCfgNode {
-  DillLoadsCall() { this = API::moduleImport("dill").getMember("loads").getACall() }
+private module Dill {
+  /**
+   * A call to `dill.load`
+   * See https://pypi.org/project/dill/ (which currently refers you
+   * to https://docs.python.org/3/library/pickle.html#pickle.load)
+   */
+  private class DillLoadCall extends Decoding::Range, DataFlow::CallCfgNode {
+    DillLoadCall() { this = API::moduleImport("dill").getMember("load").getACall() }
 
-  override predicate mayExecuteInput() { any() }
+    override predicate mayExecuteInput() { any() }
 
-  override DataFlow::Node getAnInput() { result = this.getArg(0) }
+    override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("file")] }
 
-  override DataFlow::Node getOutput() { result = this }
+    override DataFlow::Node getOutput() { result = this }
 
-  override string getFormat() { result = "dill" }
+    override string getFormat() { result = "dill" }
+  }
+
+  /**
+   * A call to `dill.loads`
+   * See https://pypi.org/project/dill/ (which currently refers you
+   * to https://docs.python.org/3/library/pickle.html#pickle.loads)
+   */
+  private class DillLoadsCall extends Decoding::Range, DataFlow::CallCfgNode {
+    DillLoadsCall() { this = API::moduleImport("dill").getMember("loads").getACall() }
+
+    override predicate mayExecuteInput() { any() }
+
+    override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("str")] }
+
+    override DataFlow::Node getOutput() { result = this }
+
+    override string getFormat() { result = "dill" }
+  }
 }

python/ql/lib/semmle/python/frameworks/Django.qll

@@ -1844,11 +1844,13 @@ private module PrivateDjango {
       t.start() and
       result.asCfgNode().(CallNode).getFunction() = this.asViewRef().asCfgNode()
       or
-      exists(DataFlow::TypeTracker t2 | result = asViewResult(t2).track(t2, t))
+      exists(DataFlow::TypeTracker t2 | result = this.asViewResult(t2).track(t2, t))
     }
 
     /** Gets a reference to the result of calling the `as_view` classmethod of this class. */
-    DataFlow::Node asViewResult() { asViewResult(DataFlow::TypeTracker::end()).flowsTo(result) }
+    DataFlow::Node asViewResult() {
+      this.asViewResult(DataFlow::TypeTracker::end()).flowsTo(result)
+    }
   }
 
   /** A class that we consider a django View class. */
@@ -1944,10 +1946,10 @@ private module PrivateDjango {
     abstract DataFlow::Node getViewArg();
 
     final override DjangoRouteHandler getARequestHandler() {
-      poorMansFunctionTracker(result) = getViewArg()
+      poorMansFunctionTracker(result) = this.getViewArg()
       or
       exists(DjangoViewClass vc |
-        getViewArg() = vc.asViewResult() and
+        this.getViewArg() = vc.asViewResult() and
         result = vc.getARequestHandler()
       )
     }

python/ql/lib/semmle/python/frameworks/Flask.qll

@@ -292,12 +292,12 @@ module Flask {
 
     override Function getARequestHandler() {
       exists(DataFlow::LocalSourceNode func_src |
-        func_src.flowsTo(getViewArg()) and
+        func_src.flowsTo(this.getViewArg()) and
         func_src.asExpr().(CallableExpr) = result.getDefinition()
       )
       or
       exists(FlaskViewClass vc |
-        getViewArg() = vc.asViewResult().getAUse() and
+        this.getViewArg() = vc.asViewResult().getAUse() and
         result = vc.getARequestHandler()
       )
     }

python/ql/lib/semmle/python/frameworks/SqlAlchemy.qll

@@ -300,10 +300,9 @@ module SqlAlchemy {
    *   there are many many constructs we would need to have models for. (see the 2
    *   examples below)
    *
-   * So instead we flag user-input to a TextClause with its' own query
-   * (`py/sqlalchemy-textclause-injection`). And so we don't highlight any parts of an
-   * ORM constructed query such as these as containing SQL, and don't need the additional
-   * taint steps either.
+   * So instead we extended the SQL injection query to include TextClause construction
+   * as a sink. And so we don't highlight any parts of an ORM constructed query such as
+   * these as containing SQL, and don't need the additional taint steps either.
    *
    * See
    * - https://docs.sqlalchemy.org/en/14/core/sqlelement.html#sqlalchemy.sql.expression.TextClause.

python/ql/lib/semmle/python/frameworks/Stdlib.qll

@@ -195,6 +195,101 @@ private module StdlibPrivate {
     }
   }
 
+  /**
+   * The `os.path` module offers a number of methods for checking if a file exists and/or has certain
+   * properties, leading to a file system access.
+   * A call to `os.path.exists` or `os.path.lexists` will check if a file exists on the file system.
+   * (Although, on some platforms, the check may return `false` due to missing permissions.)
+   * A call to `os.path.getatime` will raise `OSError` if the file does not exist or is inaccessible.
+   * See:
+   * - https://docs.python.org/3/library/os.path.html#os.path.exists
+   * - https://docs.python.org/3/library/os.path.html#os.path.lexists
+   * - https://docs.python.org/3/library/os.path.html#os.path.isfile
+   * - https://docs.python.org/3/library/os.path.html#os.path.isdir
+   * - https://docs.python.org/3/library/os.path.html#os.path.islink
+   * - https://docs.python.org/3/library/os.path.html#os.path.ismount
+   * - https://docs.python.org/3/library/os.path.html#os.path.getatime
+   * - https://docs.python.org/3/library/os.path.html#os.path.getmtime
+   * - https://docs.python.org/3/library/os.path.html#os.path.getctime
+   * - https://docs.python.org/3/library/os.path.html#os.path.getsize
+   * - https://docs.python.org/3/library/os.path.html#os.path.realpath
+   */
+  private class OsPathProbingCall extends FileSystemAccess::Range, DataFlow::CallCfgNode {
+    OsPathProbingCall() {
+      this =
+        os::path()
+            .getMember([
+                // these check if the file exists
+                "exists", "lexists", "isfile", "isdir", "islink", "ismount",
+                // these raise errors if the file does not exist
+                "getatime", "getmtime", "getctime", "getsize"
+              ])
+            .getACall()
+    }
+
+    override DataFlow::Node getAPathArgument() {
+      result in [this.getArg(0), this.getArgByName("path")]
+    }
+  }
+
+  /** A call to `os.path.samefile` will raise an exception if an `os.stat()` call on either pathname fails. */
+  private class OsPathSamefileCall extends FileSystemAccess::Range, DataFlow::CallCfgNode {
+    OsPathSamefileCall() { this = os::path().getMember("samefile").getACall() }
+
+    override DataFlow::Node getAPathArgument() {
+      result in [
+          this.getArg(0), this.getArgByName("path1"), this.getArg(1), this.getArgByName("path2")
+        ]
+    }
+  }
+
+  // Functions with non-standard arguments:
+  // - os.path.join(path, *paths)
+  // - os.path.relpath(path, start=os.curdir)
+  // these functions need special treatment when computing `getPathArg`.
+  //
+  // Functions that excluded because they can act as sanitizers:
+  // - os.path.commonpath(paths): takes a sequence
+  // - os.path.commonprefix(list): takes a list argument
+  // unless the user control all arguments, we are comparing with a known value.
+  private string pathComputation() {
+    result in [
+        "abspath", "basename", "commonpath", "dirname", "expanduser", "expandvars", "join",
+        "normcase", "normpath", "realpath", "relpath", "split", "splitdrive", "splitext"
+      ]
+  }
+
+  /**
+   * The `os.path` module offers a number of methods for computing new paths from existing paths.
+   * These should all propagate taint.
+   */
+  private class OsPathComputation extends DataFlow::CallCfgNode {
+    string methodName;
+
+    OsPathComputation() {
+      methodName = pathComputation() and
+      this = os::path().getMember(methodName).getACall()
+    }
+
+    DataFlow::Node getPathArg() {
+      result in [this.getArg(0), this.getArgByName("path")]
+      or
+      methodName = "join" and result = this.getArg(_)
+      or
+      methodName = "relpath" and result in [this.getArg(1), this.getArgByName("start")]
+    }
+  }
+
+  /** An additional taint step for path computations. */
+  private class OsPathComputationAdditionalTaintStep extends TaintTracking::AdditionalTaintStep {
+    override predicate step(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
+      exists(OsPathComputation call |
+        nodeTo = call and
+        nodeFrom = call.getPathArg()
+      )
+    }
+  }
+
   /**
    * A call to `os.path.normpath`.
    * See https://docs.python.org/3/library/os.path.html#os.path.normpath
@@ -205,16 +300,6 @@ private module StdlibPrivate {
     DataFlow::Node getPathArg() { result in [this.getArg(0), this.getArgByName("path")] }
   }
 
-  /** An additional taint step for calls to `os.path.normpath` */
-  private class OsPathNormpathCallAdditionalTaintStep extends TaintTracking::AdditionalTaintStep {
-    override predicate step(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
-      exists(OsPathNormpathCall call |
-        nodeTo = call and
-        nodeFrom = call.getPathArg()
-      )
-    }
-  }
-
   /**
    * A call to `os.path.abspath`.
    * See https://docs.python.org/3/library/os.path.html#os.path.abspath
@@ -225,16 +310,6 @@ private module StdlibPrivate {
     DataFlow::Node getPathArg() { result in [this.getArg(0), this.getArgByName("path")] }
   }
 
-  /** An additional taint step for calls to `os.path.abspath` */
-  private class OsPathAbspathCallAdditionalTaintStep extends TaintTracking::AdditionalTaintStep {
-    override predicate step(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
-      exists(OsPathAbspathCall call |
-        nodeTo = call and
-        nodeFrom = call.getPathArg()
-      )
-    }
-  }
-
   /**
    * A call to `os.path.realpath`.
    * See https://docs.python.org/3/library/os.path.html#os.path.realpath
@@ -245,16 +320,6 @@ private module StdlibPrivate {
     DataFlow::Node getPathArg() { result in [this.getArg(0), this.getArgByName("path")] }
   }
 
-  /** An additional taint step for calls to `os.path.realpath` */
-  private class OsPathRealpathCallAdditionalTaintStep extends TaintTracking::AdditionalTaintStep {
-    override predicate step(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
-      exists(OsPathRealpathCall call |
-        nodeTo = call and
-        nodeFrom = call.getPathArg()
-      )
-    }
-  }
-
   /**
    * A call to `os.system`.
    * See https://docs.python.org/3/library/os.html#os.system
@@ -397,8 +462,8 @@ private module StdlibPrivate {
       result = this.get_executable_arg()
       or
       exists(DataFlow::Node arg_args, boolean shell |
-        arg_args = get_args_arg() and
-        shell = get_shell_arg_value()
+        arg_args = this.get_args_arg() and
+        shell = this.get_shell_arg_value()
       |
         // When "executable" argument is set, and "shell" argument is `False`, the
         // "args" argument will only be used to set the program name and arguments to
@@ -428,6 +493,22 @@ private module StdlibPrivate {
   // ---------------------------------------------------------------------------
   // marshal
   // ---------------------------------------------------------------------------
+  /**
+   * A call to `marshal.load`
+   * See https://docs.python.org/3/library/marshal.html#marshal.load
+   */
+  private class MarshalLoadCall extends Decoding::Range, DataFlow::CallCfgNode {
+    MarshalLoadCall() { this = API::moduleImport("marshal").getMember("load").getACall() }
+
+    override predicate mayExecuteInput() { any() }
+
+    override DataFlow::Node getAnInput() { result = this.getArg(0) }
+
+    override DataFlow::Node getOutput() { result = this }
+
+    override string getFormat() { result = "marshal" }
+  }
+
   /**
    * A call to `marshal.loads`
    * See https://docs.python.org/3/library/marshal.html#marshal.loads
@@ -447,15 +528,23 @@ private module StdlibPrivate {
   // ---------------------------------------------------------------------------
   // pickle
   // ---------------------------------------------------------------------------
-  /** Gets a reference to the `pickle` module. */
-  DataFlow::Node pickle() { result = API::moduleImport(["pickle", "cPickle", "_pickle"]).getAUse() }
+  /** Gets a reference to any of the `pickle` modules. */
+  API::Node pickle() { result = API::moduleImport(["pickle", "cPickle", "_pickle"]) }
 
-  /** Provides models for the `pickle` module. */
-  module pickle {
-    /** Gets a reference to the `pickle.loads` function. */
-    DataFlow::Node loads() {
-      result = API::moduleImport(["pickle", "cPickle", "_pickle"]).getMember("loads").getAUse()
-    }
+  /**
+   * A call to `pickle.load`
+   * See https://docs.python.org/3/library/pickle.html#pickle.load
+   */
+  private class PickleLoadCall extends Decoding::Range, DataFlow::CallCfgNode {
+    PickleLoadCall() { this = pickle().getMember("load").getACall() }
+
+    override predicate mayExecuteInput() { any() }
+
+    override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("file")] }
+
+    override DataFlow::Node getOutput() { result = this }
+
+    override string getFormat() { result = "pickle" }
   }
 
   /**
@@ -463,11 +552,63 @@ private module StdlibPrivate {
    * See https://docs.python.org/3/library/pickle.html#pickle.loads
    */
   private class PickleLoadsCall extends Decoding::Range, DataFlow::CallCfgNode {
-    PickleLoadsCall() { this.getFunction() = pickle::loads() }
+    PickleLoadsCall() { this = pickle().getMember("loads").getACall() }
 
     override predicate mayExecuteInput() { any() }
 
-    override DataFlow::Node getAnInput() { result = this.getArg(0) }
+    override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("data")] }
+
+    override DataFlow::Node getOutput() { result = this }
+
+    override string getFormat() { result = "pickle" }
+  }
+
+  /**
+   * A construction of a `pickle.Unpickler`
+   * See https://docs.python.org/3/library/pickle.html#pickle.Unpickler
+   */
+  private class PickleUnpicklerCall extends Decoding::Range, DataFlow::CallCfgNode {
+    PickleUnpicklerCall() { this = pickle().getMember("Unpickler").getACall() }
+
+    override predicate mayExecuteInput() { any() }
+
+    override DataFlow::Node getAnInput() { result in [this.getArg(0), this.getArgByName("file")] }
+
+    override DataFlow::Node getOutput() { result = this.getAMethodCall("load") }
+
+    override string getFormat() { result = "pickle" }
+  }
+
+  // ---------------------------------------------------------------------------
+  // shelve
+  // ---------------------------------------------------------------------------
+  /**
+   * A call to `shelve.open`
+   * See https://docs.python.org/3/library/shelve.html#shelve.open
+   *
+   * Claiming there is decoding of the input to `shelve.open` is a bit questionable, since
+   * it's not the filename, but the contents of the file that is decoded.
+   *
+   * However, we definitely want to be able to alert if a user is able to control what
+   * file is used, since that can lead to code execution (even if that file is free of
+   * path injection).
+   *
+   * So right now the best way we have of modeling this seems to be to treat the filename
+   * argument as being deserialized...
+   */
+  private class ShelveOpenCall extends Decoding::Range, FileSystemAccess::Range,
+    DataFlow::CallCfgNode {
+    ShelveOpenCall() { this = API::moduleImport("shelve").getMember("open").getACall() }
+
+    override predicate mayExecuteInput() { any() }
+
+    override DataFlow::Node getAnInput() {
+      result in [this.getArg(0), this.getArgByName("filename")]
+    }
+
+    override DataFlow::Node getAPathArgument() {
+      result in [this.getArg(0), this.getArgByName("filename")]
+    }
 
     override DataFlow::Node getOutput() { result = this }
 
@@ -1136,7 +1277,7 @@ private module StdlibPrivate {
   /**
    * Gets a name of an attribute of a `pathlib.Path` object that is also a `pathlib.Path` object.
    */
-  private string pathlibPathAttribute() { result in ["parent"] }
+  private string pathlibPathAttribute() { result = "parent" }
 
   /**
    * Gets a name of a method of a `pathlib.Path` object that returns a `pathlib.Path` object.
@@ -1495,6 +1636,119 @@ private module StdlibPrivate {
       result = this.getArg(any(int i | i >= msgIndex))
     }
   }
+
+  // ---------------------------------------------------------------------------
+  // re
+  // ---------------------------------------------------------------------------
+  /**
+   * List of methods in the `re` module immediately executing a regular expression.
+   *
+   * See https://docs.python.org/3/library/re.html#module-contents
+   */
+  private class RegexExecutionMethod extends string {
+    RegexExecutionMethod() {
+      this in ["match", "fullmatch", "search", "split", "findall", "finditer", "sub", "subn"]
+    }
+
+    /** Gets the index of the argument representing the string to be searched by a regex. */
+    int getStringArgIndex() {
+      this in ["match", "fullmatch", "search", "split", "findall", "finditer"] and
+      result = 1
+      or
+      this in ["sub", "subn"] and
+      result = 2
+    }
+  }
+
+  /**
+   * A a call to a method from the `re` module immediately executing a regular expression.
+   *
+   * See `RegexExecutionMethods`
+   */
+  private class DirectRegexExecution extends DataFlow::CallCfgNode, RegexExecution::Range {
+    RegexExecutionMethod method;
+
+    DirectRegexExecution() { this = API::moduleImport("re").getMember(method).getACall() }
+
+    override DataFlow::Node getRegex() { result in [this.getArg(0), this.getArgByName("pattern")] }
+
+    override DataFlow::Node getString() {
+      result in [this.getArg(method.getStringArgIndex()), this.getArgByName("string")]
+    }
+
+    override string getName() { result = "re." + method }
+  }
+
+  /** Helper module for tracking compiled regexes. */
+  private module CompiledRegexes {
+    private DataFlow::TypeTrackingNode compiledRegex(DataFlow::TypeTracker t, DataFlow::Node regex) {
+      t.start() and
+      result = API::moduleImport("re").getMember("compile").getACall() and
+      regex in [
+          result.(DataFlow::CallCfgNode).getArg(0),
+          result.(DataFlow::CallCfgNode).getArgByName("pattern")
+        ]
+      or
+      exists(DataFlow::TypeTracker t2 | result = compiledRegex(t2, regex).track(t2, t))
+    }
+
+    DataFlow::Node compiledRegex(DataFlow::Node regex) {
+      compiledRegex(DataFlow::TypeTracker::end(), regex).flowsTo(result)
+    }
+  }
+
+  private import CompiledRegexes
+
+  /**
+   * A call on compiled regular expression (obtained via `re.compile`) executing a
+   * regular expression.
+   *
+   * Given the following example:
+   *
+   * ```py
+   * pattern = re.compile(input)
+   * pattern.match(s)
+   * ```
+   *
+   * This class will identify that `re.compile` compiles `input` and afterwards
+   * executes `re`'s `match`. As a result, `this` will refer to `pattern.match(s)`
+   * and `this.getRegexNode()` will return the node for `input` (`re.compile`'s first argument).
+   *
+   *
+   * See `RegexExecutionMethods`
+   *
+   * See https://docs.python.org/3/library/re.html#regular-expression-objects
+   */
+  private class CompiledRegexExecution extends DataFlow::MethodCallNode, RegexExecution::Range {
+    DataFlow::Node regexNode;
+    RegexExecutionMethod method;
+
+    CompiledRegexExecution() { this.calls(compiledRegex(regexNode), method) }
+
+    override DataFlow::Node getRegex() { result = regexNode }
+
+    override DataFlow::Node getString() {
+      result in [this.getArg(method.getStringArgIndex() - 1), this.getArgByName("string")]
+    }
+
+    override string getName() { result = "re." + method }
+  }
+
+  /**
+   * A call to 're.escape'.
+   * See https://docs.python.org/3/library/re.html#re.escape
+   */
+  private class ReEscapeCall extends Escaping::Range, DataFlow::CallCfgNode {
+    ReEscapeCall() { this = API::moduleImport("re").getMember("escape").getACall() }
+
+    override DataFlow::Node getAnInput() {
+      result in [this.getArg(0), this.getArgByName("pattern")]
+    }
+
+    override DataFlow::Node getOutput() { result = this }
+
+    override string getKind() { result = Escaping::getRegexKind() }
+  }
 }
 
 // ---------------------------------------------------------------------------

python/ql/lib/semmle/python/frameworks/Tornado.qll

@@ -318,7 +318,7 @@ private module Tornado {
               ]
           }
 
-          override string getMethodName() { result in ["full_url"] }
+          override string getMethodName() { result = "full_url" }
 
           override string getAsyncMethodName() { none() }
         }

python/ql/lib/semmle/python/frameworks/Werkzeug.qll

@@ -58,7 +58,7 @@ module Werkzeug {
 
       override string getAttributeName() { none() }
 
-      override string getMethodName() { result in ["getlist"] }
+      override string getMethodName() { result = "getlist" }
 
       override string getAsyncMethodName() { none() }
     }

python/ql/lib/semmle/python/frameworks/Yarl.qll

@@ -68,7 +68,7 @@ module Yarl {
           ]
       }
 
-      override string getMethodName() { result in ["human_repr"] }
+      override string getMethodName() { result = "human_repr" }
 
       override string getAsyncMethodName() { none() }
     }

python/ql/lib/semmle/python/objects/ObjectAPI.qll

@@ -79,7 +79,7 @@ class Value extends TObject {
    * The location spans column `startcolumn` of line `startline` to
    * column `endcolumn` of line `endline` in file `filepath`.
    * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn

python/ql/lib/semmle/python/objects/TObject.qll

@@ -387,7 +387,7 @@ private predicate concrete_class(PythonClassObjectInternal cls) {
       not exists(Raise r, Name ex |
         r.getScope() = f and
         (r.getException() = ex or r.getException().(Call).getFunc() = ex) and
-        (ex.getId() = "NotImplementedError" or ex.getId() = "NotImplemented")
+        ex.getId() = ["NotImplementedError", "NotImplemented"]
       )
     )
   )
@@ -437,11 +437,7 @@ predicate missing_imported_module(ControlFlowNode imp, Context ctx, string name)
  * Helper for missing modules to determine if name `x.y` is a module `x.y` or
  * an attribute `y` of module `x`. This list should be added to as required.
  */
-predicate common_module_name(string name) {
-  name = "zope.interface"
-  or
-  name = "six.moves"
-}
+predicate common_module_name(string name) { name = ["zope.interface", "six.moves"] }
 
 /**
  * A declaration of a class, either a built-in class or a source definition
@@ -482,16 +478,11 @@ library class ClassDecl extends @py_object {
    */
   predicate isSpecial() {
     exists(string name | this = Builtin::special(name) |
-      name = "type" or
-      name = "super" or
-      name = "bool" or
-      name = "NoneType" or
-      name = "tuple" or
-      name = "property" or
-      name = "ClassMethod" or
-      name = "StaticMethod" or
-      name = "MethodType" or
-      name = "ModuleType"
+      name =
+        [
+          "type", "super", "bool", "NoneType", "tuple", "property", "ClassMethod", "StaticMethod",
+          "MethodType", "ModuleType"
+        ]
     )
   }
 
@@ -514,11 +505,7 @@ library class ClassDecl extends @py_object {
 
   /** Holds if this class is the abstract base class */
   predicate isAbstractBaseClass(string name) {
-    exists(Module m |
-      m.getName() = "_abcoll"
-      or
-      m.getName() = "_collections_abc"
-    |
+    exists(Module m | m.getName() = ["_abcoll", "_collections_abc"] |
       this.getClass().getScope() = m and
       this.getName() = name
     )

python/ql/lib/semmle/python/pointsto/PointsTo.qll

@@ -300,7 +300,7 @@ module PointsToInternal {
     ssa_definition_points_to(var.getDefinition(), context, value, origin)
     or
     exists(EssaVariable prev |
-      ssaShortCut+(prev, var) and
+      ssaShortCut(prev, var) and
       variablePointsTo(prev, context, value, origin)
     )
   }

python/ql/lib/semmle/python/regex.qll

@@ -102,15 +102,7 @@ string mode_from_node(DataFlow::Node node) { node = re_flag_tracker(result) }
  * Gets a regular expression mode flag associated with the given value.
  */
 deprecated string mode_from_mode_object(Value obj) {
-  (
-    result = "DEBUG" or
-    result = "IGNORECASE" or
-    result = "LOCALE" or
-    result = "MULTILINE" or
-    result = "DOTALL" or
-    result = "UNICODE" or
-    result = "VERBOSE"
-  ) and
+  result in ["DEBUG", "IGNORECASE", "LOCALE", "MULTILINE", "DOTALL", "UNICODE", "VERBOSE"] and
   exists(int flag |
     flag = Value::named("sre_constants.SRE_FLAG_" + result).(OI::ObjectInternal).intValue() and
     obj.(OI::ObjectInternal).intValue().bitAnd(flag) = flag
@@ -612,14 +604,7 @@ abstract class RegexString extends Expr {
     this.getChar(start + 1) = "?" and
     end = start + 3 and
     c = this.getChar(start + 2) and
-    (
-      c = "i" or
-      c = "L" or
-      c = "m" or
-      c = "s" or
-      c = "u" or
-      c = "x"
-    )
+    c in ["i", "L", "m", "s", "u", "x"]
   }
 
   /**

python/ql/lib/semmle/python/security/ClearText.qll

@@ -47,11 +47,7 @@ module ClearTextLogging {
         meth.getObject(name).(NameNode).getId().matches("logg%") and
         call.getAnArg() = this
       |
-        name = "error" or
-        name = "warn" or
-        name = "warning" or
-        name = "debug" or
-        name = "info"
+        name = ["error", "warn", "warning", "debug", "info"]
       )
     }
   }

python/ql/lib/semmle/python/security/Exceptions.qll

@@ -74,13 +74,10 @@ class ExceptionInfoSequence extends SequenceKind {
 class CallToTracebackFunction extends ErrorInfoSource {
   CallToTracebackFunction() {
     exists(string name |
-      name = "extract_tb" or
-      name = "extract_stack" or
-      name = "format_list" or
-      name = "format_exception_only" or
-      name = "format_exception" or
-      name = "format_tb" or
-      name = "format_stack"
+      name in [
+          "extract_tb", "extract_stack", "format_list", "format_exception_only", "format_exception",
+          "format_tb", "format_stack"
+        ]
     |
       this = traceback_function(name).getACall()
     )

python/ql/lib/semmle/python/security/dataflow/ChainedConfigs12.qll

@@ -46,7 +46,7 @@ class CustomPathNode extends TCustomPathNode {
    * The location spans column `startcolumn` of line `startline` to
    * column `endcolumn` of line `endline` in file `filepath`.
    * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn

python/ql/lib/semmle/python/security/dataflow/PolynomialReDoSCustomizations.qll

@@ -60,8 +60,8 @@ module PolynomialReDoS {
     RegExpTerm t;
 
     RegexExecutionAsSink() {
-      exists(CompiledRegexes::RegexExecution re |
-        re.getRegexNode().asExpr() = t.getRegex() and
+      exists(RegexExecution re |
+        re.getRegex().asExpr() = t.getRegex() and
         this = re.getString()
       ) and
       t.isRootTerm()
@@ -76,137 +76,3 @@ module PolynomialReDoS {
    */
   class StringConstCompareAsSanitizerGuard extends SanitizerGuard, StringConstCompare { }
 }
-
-/** Helper module for tracking compiled regexes. */
-private module CompiledRegexes {
-  // TODO: This module should be refactored and merged with the experimental work done on detecting
-  // regex injections, such that this can be expressed from just using a concept.
-  /** A configuration for finding uses of compiled regexes. */
-  class RegexDefinitionConfiguration extends DataFlow2::Configuration {
-    RegexDefinitionConfiguration() { this = "RegexDefinitionConfiguration" }
-
-    override predicate isSource(DataFlow::Node source) { source instanceof RegexDefinitonSource }
-
-    override predicate isSink(DataFlow::Node sink) { sink instanceof RegexDefinitionSink }
-  }
-
-  /** A regex compilation. */
-  class RegexDefinitonSource extends DataFlow::CallCfgNode {
-    DataFlow::Node regexNode;
-
-    RegexDefinitonSource() {
-      this = API::moduleImport("re").getMember("compile").getACall() and
-      regexNode in [this.getArg(0), this.getArgByName("pattern")]
-    }
-
-    /** Gets the regex that is being compiled by this node. */
-    RegExpTerm getRegExp() { result.getRegex() = regexNode.asExpr() and result.isRootTerm() }
-
-    /** Gets the data flow node for the regex being compiled by this node. */
-    DataFlow::Node getRegexNode() { result = regexNode }
-  }
-
-  /** A use of a compiled regex. */
-  class RegexDefinitionSink extends DataFlow::Node {
-    RegexExecutionMethod method;
-    DataFlow::CallCfgNode executingCall;
-
-    RegexDefinitionSink() {
-      exists(DataFlow::AttrRead reMethod |
-        executingCall.getFunction() = reMethod and
-        reMethod.getAttributeName() = method and
-        this = reMethod.getObject()
-      )
-    }
-
-    /** Gets the method used to execute the regex. */
-    RegexExecutionMethod getMethod() { result = method }
-
-    /** Gets the data flow node for the executing call. */
-    DataFlow::CallCfgNode getExecutingCall() { result = executingCall }
-  }
-
-  /** A data flow node executing a regex. */
-  abstract class RegexExecution extends DataFlow::Node {
-    /** Gets the data flow node for the regex being compiled by this node. */
-    abstract DataFlow::Node getRegexNode();
-
-    /** Gets a dataflow node for the string to be searched or matched against. */
-    abstract DataFlow::Node getString();
-  }
-
-  private class RegexExecutionMethod extends string {
-    RegexExecutionMethod() {
-      this in ["match", "fullmatch", "search", "split", "findall", "finditer", "sub", "subn"]
-    }
-  }
-
-  /** Gets the index of the argument representing the string to be searched by a regex. */
-  int stringArg(RegexExecutionMethod method) {
-    method in ["match", "fullmatch", "search", "split", "findall", "finditer"] and
-    result = 1
-    or
-    method in ["sub", "subn"] and
-    result = 2
-  }
-
-  /**
-   * A class to find `re` methods immediately executing an expression.
-   *
-   * See `RegexExecutionMethods`
-   */
-  class DirectRegex extends DataFlow::CallCfgNode, RegexExecution {
-    RegexExecutionMethod method;
-
-    DirectRegex() { this = API::moduleImport("re").getMember(method).getACall() }
-
-    override DataFlow::Node getRegexNode() {
-      result in [this.getArg(0), this.getArgByName("pattern")]
-    }
-
-    override DataFlow::Node getString() {
-      result in [this.getArg(stringArg(method)), this.getArgByName("string")]
-    }
-  }
-
-  /**
-   * A class to find `re` methods immediately executing a compiled expression by `re.compile`.
-   *
-   * Given the following example:
-   *
-   * ```py
-   * pattern = re.compile(input)
-   * pattern.match(s)
-   * ```
-   *
-   * This class will identify that `re.compile` compiles `input` and afterwards
-   * executes `re`'s `match`. As a result, `this` will refer to `pattern.match(s)`
-   * and `this.getRegexNode()` will return the node for `input` (`re.compile`'s first argument)
-   *
-   *
-   * See `RegexExecutionMethods`
-   *
-   * See https://docs.python.org/3/library/re.html#regular-expression-objects
-   */
-  private class CompiledRegex extends DataFlow::CallCfgNode, RegexExecution {
-    DataFlow::Node regexNode;
-    RegexExecutionMethod method;
-
-    CompiledRegex() {
-      exists(
-        RegexDefinitionConfiguration conf, RegexDefinitonSource source, RegexDefinitionSink sink
-      |
-        conf.hasFlow(source, sink) and
-        regexNode = source.getRegexNode() and
-        method = sink.getMethod() and
-        this = sink.getExecutingCall()
-      )
-    }
-
-    override DataFlow::Node getRegexNode() { result = regexNode }
-
-    override DataFlow::Node getString() {
-      result in [this.getArg(stringArg(method) - 1), this.getArgByName("string")]
-    }
-  }
-}

python/ql/lib/semmle/python/security/dataflow/ReflectedXSSCustomizations.qll

@@ -59,7 +59,7 @@ module ReflectedXSS {
   class HtmlEscapingAsSanitizer extends Sanitizer {
     HtmlEscapingAsSanitizer() {
       // TODO: For now, since there is not an `isSanitizingStep` member-predicate part of a
-      // `TaintTracking::Configuration`, we use treat the output is a taint-sanitizer. This
+      // `TaintTracking::Configuration`, we treat the output as a taint-sanitizer. This
       // is slightly imprecise, which you can see in the `m_unsafe + SAFE` test-case in
       // python/ql/test/library-tests/frameworks/markupsafe/taint_test.py
       //

python/ql/lib/semmle/python/security/dataflow/SQLAlchemyTextClauseCustomizations.qll

@@ -1,56 +0,0 @@
-/**
- * Provides default sources, sinks and sanitizers for detecting
- * "SQLAlchemy TextClause injection"
- * vulnerabilities, as well as extension points for adding your own.
- */
-
-private import python
-private import semmle.python.dataflow.new.DataFlow
-private import semmle.python.Concepts
-private import semmle.python.dataflow.new.RemoteFlowSources
-private import semmle.python.dataflow.new.BarrierGuards
-private import semmle.python.frameworks.SqlAlchemy
-
-/**
- * Provides default sources, sinks and sanitizers for detecting
- * "SQLAlchemy TextClause injection"
- * vulnerabilities, as well as extension points for adding your own.
- */
-module SQLAlchemyTextClause {
-  /**
-   * A data flow source for "SQLAlchemy TextClause injection" vulnerabilities.
-   */
-  abstract class Source extends DataFlow::Node { }
-
-  /**
-   * A data flow sink for "SQLAlchemy TextClause injection" vulnerabilities.
-   */
-  abstract class Sink extends DataFlow::Node { }
-
-  /**
-   * A sanitizer for "SQLAlchemy TextClause injection" vulnerabilities.
-   */
-  abstract class Sanitizer extends DataFlow::Node { }
-
-  /**
-   * A sanitizer guard for "SQLAlchemy TextClause injection" vulnerabilities.
-   */
-  abstract class SanitizerGuard extends DataFlow::BarrierGuard { }
-
-  /**
-   * A source of remote user input, considered as a flow source.
-   */
-  class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { }
-
-  /**
-   * The text argument of a SQLAlchemy TextClause construction, considered as a flow sink.
-   */
-  class TextArgAsSink extends Sink {
-    TextArgAsSink() { this = any(SqlAlchemy::TextClause::TextClauseConstruction tcc).getTextArg() }
-  }
-
-  /**
-   * A comparison with a constant string, considered as a sanitizer-guard.
-   */
-  class StringConstCompareAsSanitizerGuard extends SanitizerGuard, StringConstCompare { }
-}

python/ql/lib/semmle/python/security/dataflow/SqlInjectionCustomizations.qll

@@ -9,6 +9,7 @@ private import semmle.python.dataflow.new.DataFlow
 private import semmle.python.Concepts
 private import semmle.python.dataflow.new.RemoteFlowSources
 private import semmle.python.dataflow.new.BarrierGuards
+private import semmle.python.frameworks.SqlAlchemy
 
 /**
  * Provides default sources, sinks and sanitizers for detecting
@@ -48,6 +49,13 @@ module SqlInjection {
     SqlExecutionAsSink() { this = any(SqlExecution e).getSql() }
   }
 
+  /**
+   * The text argument of a SQLAlchemy TextClause construction, considered as a flow sink.
+   */
+  class TextArgAsSink extends Sink {
+    TextArgAsSink() { this = any(SqlAlchemy::TextClause::TextClauseConstruction tcc).getTextArg() }
+  }
+
   /**
    * A comparison with a constant string, considered as a sanitizer-guard.
    */

python/ql/lib/semmle/python/security/injection/Command.qll

@@ -13,18 +13,11 @@ import semmle.python.security.strings.Untrusted
 /** Abstract taint sink that is potentially vulnerable to malicious shell commands. */
 abstract class CommandSink extends TaintSink { }
 
-private ModuleObject osOrPopenModule() {
-  result.getName() = "os" or
-  result.getName() = "popen2"
-}
+private ModuleObject osOrPopenModule() { result.getName() = ["os", "popen2"] }
 
 private Object makeOsCall() {
   exists(string name | result = ModuleObject::named("subprocess").attr(name) |
-    name = "Popen" or
-    name = "call" or
-    name = "check_call" or
-    name = "check_output" or
-    name = "run"
+    name = ["Popen", "call", "check_call", "check_output", "run"]
   )
 }
 
@@ -65,8 +58,7 @@ class ShellCommand extends CommandSink {
       call.getAnArg() = this and
       call.getFunction().refersTo(osOrPopenModule().attr(name))
     |
-      name = "system" or
-      name = "popen" or
+      name = ["system", "popen"] or
       name.matches("popen_")
     )
     or

python/ql/lib/semmle/python/security/injection/RegexInjection.qll

@@ -1,9 +1,10 @@
 /**
- * Provides a taint-tracking configuration for detecting "SQLAlchemy TextClause injection" vulnerabilities.
+ * Provides a taint-tracking configuration for detecting regular expression injection
+ * vulnerabilities.
  *
  * Note, for performance reasons: only import this file if
- * `SQLAlchemyTextClause::Configuration` is needed, otherwise
- * `SQLAlchemyTextClauseCustomizations` should be imported instead.
+ * `RegexInjection::Configuration` is needed, otherwise
+ * `RegexInjectionCustomizations` should be imported instead.
  */
 
 private import python
@@ -11,16 +12,17 @@ import semmle.python.dataflow.new.DataFlow
 import semmle.python.dataflow.new.TaintTracking
 
 /**
- * Provides a taint-tracking configuration for detecting "SQLAlchemy TextClause injection" vulnerabilities.
+ * Provides a taint-tracking configuration for detecting regular expression injection
+ * vulnerabilities.
  */
-module SQLAlchemyTextClause {
-  import SQLAlchemyTextClauseCustomizations::SQLAlchemyTextClause
+module RegexInjection {
+  import RegexInjectionCustomizations::RegexInjection
 
   /**
-   * A taint-tracking configuration for detecting "SQLAlchemy TextClause injection" vulnerabilities.
+   * A taint-tracking configuration for detecting "reflected server-side cross-site scripting" vulnerabilities.
    */
   class Configuration extends TaintTracking::Configuration {
-    Configuration() { this = "SQLAlchemyTextClause" }
+    Configuration() { this = "RegexInjection" }
 
     override predicate isSource(DataFlow::Node source) { source instanceof Source }
 

python/ql/lib/semmle/python/security/injection/RegexInjectionCustomizations.qll

@@ -0,0 +1,62 @@
+/**
+ * Provides default sources, sinks and sanitizers for detecting
+ * "regular expression injection"
+ * vulnerabilities, as well as extension points for adding your own.
+ */
+
+private import python
+private import semmle.python.Concepts
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.dataflow.new.TaintTracking
+private import semmle.python.dataflow.new.RemoteFlowSources
+
+/**
+ * Provides default sources, sinks and sanitizers for detecting
+ * "regular expression injection"
+ * vulnerabilities, as well as extension points for adding your own.
+ */
+module RegexInjection {
+  /**
+   * A data flow source for "regular expression injection" vulnerabilities.
+   */
+  abstract class Source extends DataFlow::Node { }
+
+  /**
+   * A sink for "regular expression injection" vulnerabilities is the execution of a regular expression.
+   * If you have a custom way to execute regular expressions, you can extend `RegexExecution::Range`.
+   */
+  class Sink extends DataFlow::Node {
+    RegexExecution regexExecution;
+
+    Sink() { this = regexExecution.getRegex() }
+
+    /** Gets the call that executes the regular expression marked by this sink. */
+    RegexExecution getRegexExecution() { result = regexExecution }
+  }
+
+  /**
+   * A sanitizer for "regular expression injection" vulnerabilities.
+   */
+  abstract class Sanitizer extends DataFlow::Node { }
+
+  /**
+   * A sanitizer guard for "regular expression injection" vulnerabilities.
+   */
+  abstract class SanitizerGuard extends DataFlow::BarrierGuard { }
+
+  /**
+   * A source of remote user input, considered as a flow source.
+   */
+  class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { }
+
+  /**
+   * A regex escaping, considered as a sanitizer.
+   */
+  class RegexEscapingAsSanitizer extends Sanitizer {
+    RegexEscapingAsSanitizer() {
+      // Due to use-use flow, we want the output rather than an input
+      // (so the input can still flow to other sinks).
+      this = any(RegexEscaping esc).getOutput()
+    }
+  }
+}

python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll

@@ -139,8 +139,6 @@ class RegExpRoot extends RegExpTerm {
   predicate isRelevant() {
     // there is at least one repetition
     getRoot(any(InfiniteRepetitionQuantifier q)) = this and
-    // there are no lookbehinds
-    not exists(RegExpLookbehind lbh | getRoot(lbh) = this) and
     // is actually used as a RegExp
     isUsedAsRegExp() and
     // not excluded for library specific reasons
@@ -479,7 +477,7 @@ private module CharacterClasses {
       result = ["0", "9"]
       or
       cc.getValue() = "s" and
-      result = [" "]
+      result = " "
       or
       cc.getValue() = "w" and
       result = ["a", "Z", "_", "0", "9"]
@@ -492,7 +490,7 @@ private module CharacterClasses {
       result = "9"
       or
       cc.getValue() = "s" and
-      result = [" "]
+      result = " "
       or
       cc.getValue() = "w" and
       result = "a"

python/ql/lib/semmle/python/templates/PyxlTags.qll

@@ -29,7 +29,7 @@ private predicate pyxl_tag(Call c, string name) {
 }
 
 class PyxlHtmlTag extends PyxlTag {
-  PyxlHtmlTag() { this.getPyxlTagName().prefix(2) = "x_" }
+  PyxlHtmlTag() { this.getPyxlTagName().matches("x\\_%") }
 
   string getTagName() { result = this.getPyxlTagName().suffix(2) }
 

python/ql/lib/semmle/python/types/Extensions.qll

@@ -112,14 +112,7 @@ class BottleRoutePointToExtension extends PointsToExtension {
 
 /* Python 3.6+ regex module constants */
 string short_flag(string flag) {
-  (
-    flag = "ASCII" or
-    flag = "IGNORECASE" or
-    flag = "LOCALE" or
-    flag = "UNICODE" or
-    flag = "MULTILINE" or
-    flag = "TEMPLATE"
-  ) and
+  flag in ["ASCII", "IGNORECASE", "LOCALE", "UNICODE", "MULTILINE", "TEMPLATE"] and
   result = flag.prefix(1)
   or
   flag = "DOTALL" and result = "S"

python/ql/lib/semmle/python/types/FunctionObject.qll

@@ -183,6 +183,7 @@ class PyFunctionObject extends FunctionObject {
   }
 
   /** Factored out to help join ordering */
+  pragma[noinline]
   private predicate implicitlyReturns(Object none_, ClassObject noneType) {
     noneType = theNoneType() and
     not this.getFunction().isGenerator() and

python/ql/lib/semmle/python/types/Object.qll

@@ -69,7 +69,7 @@ class Object extends @py_object {
    * The location spans column `startcolumn` of line `startline` to
    * column `endcolumn` of line `endline` in file `filepath`.
    * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn

python/ql/lib/semmle/python/web/Http.qll

@@ -33,7 +33,7 @@ class WsgiEnvironment extends TaintKind {
       (
         text = "QUERY_STRING" or
         text = "PATH_INFO" or
-        text.prefix(5) = "HTTP_"
+        text.matches("HTTP\\_%")
       )
     )
   }

python/ql/lib/semmle/python/web/HttpConstants.qll

@@ -1,13 +1,5 @@
 /** Gets an HTTP verb, in upper case */
-string httpVerb() {
-  result = "GET" or
-  result = "POST" or
-  result = "PUT" or
-  result = "PATCH" or
-  result = "DELETE" or
-  result = "OPTIONS" or
-  result = "HEAD"
-}
+string httpVerb() { result in ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"] }
 
 /** Gets an HTTP verb, in lower case */
 string httpVerbLower() { result = httpVerb().toLowerCase() }

python/ql/lib/semmle/python/web/django/Model.qll

@@ -15,31 +15,11 @@ class DjangoDbTableObjects extends TaintKind {
 
   override TaintKind getTaintOfMethodResult(string name) {
     result = this and
-    (
-      name = "filter" or
-      name = "exclude" or
-      name = "annotate" or
-      name = "order_by" or
-      name = "reverse" or
-      name = "distinct" or
-      name = "values" or
-      name = "values_list" or
-      name = "dates" or
-      name = "datetimes" or
-      name = "none" or
-      name = "all" or
-      name = "union" or
-      name = "intersection" or
-      name = "difference" or
-      name = "select_related" or
-      name = "prefetch_related" or
-      name = "extra" or
-      name = "defer" or
-      name = "only" or
-      name = "using" or
-      name = "select_for_update" or
-      name = "raw"
-    )
+    name in [
+        "filter", "exclude", "none", "all", "union", "intersection", "difference", "select_related",
+        "prefetch_related", "extra", "defer", "only", "annotate", "using", "select_for_update",
+        "raw", "order_by", "reverse", "distinct", "values", "values_list", "dates", "datetimes"
+      ]
   }
 }
 

python/ql/lib/semmle/python/web/falcon/Request.qll

@@ -12,13 +12,7 @@ class FalconRequest extends TaintKind {
     name = "env" and result instanceof WsgiEnvironment
     or
     result instanceof ExternalStringKind and
-    (
-      name = "uri" or
-      name = "url" or
-      name = "forwarded_uri" or
-      name = "relative_uri" or
-      name = "query_string"
-    )
+    name in ["uri", "url", "forwarded_uri", "relative_uri", "query_string"]
     or
     result instanceof ExternalStringDictKind and
     (name = "cookies" or name = "params")

python/ql/lib/semmle/python/web/flask/Request.qll

@@ -32,12 +32,7 @@ class FlaskRequestData extends HttpRequestTaintSource {
 class FlaskRequestArgs extends HttpRequestTaintSource {
   FlaskRequestArgs() {
     exists(string attr | flask_request_attr(this, attr) |
-      attr = "args" or
-      attr = "form" or
-      attr = "values" or
-      attr = "files" or
-      attr = "headers" or
-      attr = "json"
+      attr in ["args", "form", "values", "files", "headers", "json"]
     )
   }
 

python/ql/lib/semmle/python/xml/XML.qll

@@ -24,7 +24,7 @@ class XMLLocatable extends @xmllocatable, TXMLLocatable {
    * The location spans column `startcolumn` of line `startline` to
    * column `endcolumn` of line `endline` in file `filepath`.
    * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn
@@ -108,7 +108,7 @@ class XMLParent extends @xmlparent {
   }
 
   /** Gets the text value contained in this XML parent. */
-  string getTextValue() { result = allCharactersString() }
+  string getTextValue() { result = this.allCharactersString() }
 
   /** Gets a printable representation of this XML parent. */
   string toString() { result = this.getName() }
@@ -119,7 +119,7 @@ class XMLFile extends XMLParent, File {
   XMLFile() { xmlEncoding(this, _) }
 
   /** Gets a printable representation of this XML file. */
-  override string toString() { result = getName() }
+  override string toString() { result = this.getName() }
 
   /** Gets the name of this XML file. */
   override string getName() { result = File.super.getAbsolutePath() }
@@ -129,14 +129,14 @@ class XMLFile extends XMLParent, File {
    *
    * Gets the path of this XML file.
    */
-  deprecated string getPath() { result = getAbsolutePath() }
+  deprecated string getPath() { result = this.getAbsolutePath() }
 
   /**
    * DEPRECATED: Use `getParentContainer().getAbsolutePath()` instead.
    *
    * Gets the path of the folder that contains this XML file.
    */
-  deprecated string getFolder() { result = getParentContainer().getAbsolutePath() }
+  deprecated string getFolder() { result = this.getParentContainer().getAbsolutePath() }
 
   /** Gets the encoding of this XML file. */
   string getEncoding() { xmlEncoding(this, result) }
@@ -200,7 +200,7 @@ class XMLDTD extends XMLLocatable, @xmldtd {
  */
 class XMLElement extends @xmlelement, XMLParent, XMLLocatable {
   /** Holds if this XML element has the given `name`. */
-  predicate hasName(string name) { name = getName() }
+  predicate hasName(string name) { name = this.getName() }
 
   /** Gets the name of this XML element. */
   override string getName() { xmlElements(this, result, _, _, _) }
@@ -239,7 +239,7 @@ class XMLElement extends @xmlelement, XMLParent, XMLLocatable {
   string getAttributeValue(string name) { result = this.getAttribute(name).getValue() }
 
   /** Gets a printable representation of this XML element. */
-  override string toString() { result = getName() }
+  override string toString() { result = this.getName() }
 }
 
 /**

python/ql/src/Classes/UselessClass.ql

@@ -52,11 +52,7 @@ predicate is_stateful(Class c) {
     call.getFunc() = a and
     a.getName() = name
   |
-    name = "pop" or
-    name = "remove" or
-    name = "discard" or
-    name = "extend" or
-    name = "append"
+    name in ["pop", "remove", "discard", "extend", "append"]
   )
 }
 

python/ql/src/Diagnostics/ExtractionErrors.ql

@@ -1,23 +0,0 @@
-/**
- * @name Python extraction errors
- * @description List all extraction errors for Python files in the source code directory.
- * @kind diagnostic
- * @id py/diagnostics/extraction-errors
- */
-
-import python
-
-/**
- * Gets the SARIF severity for errors.
- *
- * See point 3.27.10 in https://docs.oasis-open.org/sarif/sarif/v2.0/sarif-v2.0.html for
- * what error means.
- */
-int getErrorSeverity() { result = 2 }
-
-from SyntaxError error, File file
-where
-  file = error.getFile() and
-  exists(file.getRelativePath())
-select error, "Extraction failed in " + file + " with error " + error.getMessage(),
-  getErrorSeverity()

python/ql/src/Diagnostics/ExtractionWarnings.ql

@@ -0,0 +1,36 @@
+/**
+ * @name Python extraction warnings
+ * @description List all extraction warnings for Python files in the source code directory.
+ * @kind diagnostic
+ * @id py/diagnostics/extraction-warnings
+ */
+
+import python
+
+/**
+ * Gets the SARIF severity for warnings.
+ *
+ * See https://docs.oasis-open.org/sarif/sarif/v2.1.0/csprd01/sarif-v2.1.0-csprd01.html#_Toc10541338
+ */
+int getWarningSeverity() { result = 1 }
+
+// The spec
+// https://docs.oasis-open.org/sarif/sarif/v2.1.0/csprd01/sarif-v2.1.0-csprd01.html#_Toc10541338
+// defines error and warning as:
+//
+// "error": A serious problem was found. The condition encountered by the tool resulted
+// in the analysis being halted or caused the results to be incorrect or incomplete.
+//
+// "warning": A problem that is not considered serious was found. The condition
+// encountered by the tool is such that it is uncertain whether a problem occurred, or
+// is such that the analysis might be incomplete but the results that were generated are
+// probably valid.
+//
+// So SyntaxErrors are reported at the warning level, since analysis might be incomplete
+// but the results that were generated are probably valid.
+from SyntaxError error, File file
+where
+  file = error.getFile() and
+  exists(file.getRelativePath())
+select error, "Extraction failed in " + file + " with error " + error.getMessage(),
+  getWarningSeverity()

python/ql/src/Functions/IncorrectRaiseInSpecialMethod.ql

@@ -22,49 +22,14 @@ private predicate indexing_method(string name) {
 }
 
 private predicate arithmetic_method(string name) {
-  name = "__add__" or
-  name = "__sub__" or
-  name = "__div__" or
-  name = "__pos__" or
-  name = "__abs__" or
-  name = "__floordiv__" or
-  name = "__div__" or
-  name = "__divmod__" or
-  name = "__lshift__" or
-  name = "__and__" or
-  name = "__or__" or
-  name = "__xor__" or
-  name = "__rshift__" or
-  name = "__pow__" or
-  name = "__mul__" or
-  name = "__neg__" or
-  name = "__radd__" or
-  name = "__rsub__" or
-  name = "__rdiv__" or
-  name = "__rfloordiv__" or
-  name = "__rdiv__" or
-  name = "__rlshift__" or
-  name = "__rand__" or
-  name = "__ror__" or
-  name = "__rxor__" or
-  name = "__rrshift__" or
-  name = "__rpow__" or
-  name = "__rmul__" or
-  name = "__truediv__" or
-  name = "__rtruediv__" or
-  name = "__iadd__" or
-  name = "__isub__" or
-  name = "__idiv__" or
-  name = "__ifloordiv__" or
-  name = "__idiv__" or
-  name = "__ilshift__" or
-  name = "__iand__" or
-  name = "__ior__" or
-  name = "__ixor__" or
-  name = "__irshift__" or
-  name = "__ipow__" or
-  name = "__imul__" or
-  name = "__itruediv__"
+  name in [
+      "__add__", "__sub__", "__or__", "__xor__", "__rshift__", "__pow__", "__mul__", "__neg__",
+      "__radd__", "__rsub__", "__rdiv__", "__rfloordiv__", "__div__", "__rdiv__", "__rlshift__",
+      "__rand__", "__ror__", "__rxor__", "__rrshift__", "__rpow__", "__rmul__", "__truediv__",
+      "__rtruediv__", "__pos__", "__iadd__", "__isub__", "__idiv__", "__ifloordiv__", "__idiv__",
+      "__ilshift__", "__iand__", "__ior__", "__ixor__", "__irshift__", "__abs__", "__ipow__",
+      "__imul__", "__itruediv__", "__floordiv__", "__div__", "__divmod__", "__lshift__", "__and__"
+    ]
 }
 
 private predicate ordering_method(string name) {

python/ql/src/Functions/SignatureOverriddenMethod.ql

@@ -24,7 +24,6 @@ where
   not derived.getScope().isSpecialMethod() and
   derived.getName() != "__init__" and
   derived.isNormalMethod() and
-  not derived.getScope().isSpecialMethod() and
   // call to overrides distributed for efficiency
   (
     derived.overrides(base) and derived.minParameters() > base.maxParameters()

python/ql/src/Functions/SignatureSpecialMethods.ql

@@ -13,98 +13,29 @@
 import python
 
 predicate is_unary_op(string name) {
-  name = "__del__" or
-  name = "__repr__" or
-  name = "__str__" or
-  name = "__hash__" or
-  name = "__bool__" or
-  name = "__nonzero__" or
-  name = "__unicode__" or
-  name = "__len__" or
-  name = "__iter__" or
-  name = "__reversed__" or
-  name = "__neg__" or
-  name = "__pos__" or
-  name = "__abs__" or
-  name = "__invert__" or
-  name = "__complex__" or
-  name = "__int__" or
-  name = "__float__" or
-  name = "__long__" or
-  name = "__oct__" or
-  name = "__hex__" or
-  name = "__index__" or
-  name = "__enter__"
+  name in [
+      "__del__", "__repr__", "__neg__", "__pos__", "__abs__", "__invert__", "__complex__",
+      "__int__", "__float__", "__long__", "__oct__", "__hex__", "__str__", "__index__", "__enter__",
+      "__hash__", "__bool__", "__nonzero__", "__unicode__", "__len__", "__iter__", "__reversed__"
+    ]
 }
 
 predicate is_binary_op(string name) {
-  name = "__lt__" or
-  name = "__le__" or
-  name = "__eq__" or
-  name = "__ne__" or
-  name = "__gt__" or
-  name = "__ge__" or
-  name = "__cmp__" or
-  name = "__rcmp__" or
-  name = "__getattr___" or
-  name = "__getattribute___" or
-  name = "__delattr__" or
-  name = "__delete__" or
-  name = "__instancecheck__" or
-  name = "__subclasscheck__" or
-  name = "__getitem__" or
-  name = "__delitem__" or
-  name = "__contains__" or
-  name = "__add__" or
-  name = "__sub__" or
-  name = "__mul__" or
-  name = "__floordiv__" or
-  name = "__div__" or
-  name = "__truediv__" or
-  name = "__mod__" or
-  name = "__divmod__" or
-  name = "__lshift__" or
-  name = "__rshift__" or
-  name = "__and__" or
-  name = "__xor__" or
-  name = "__or__" or
-  name = "__radd__" or
-  name = "__rsub__" or
-  name = "__rmul__" or
-  name = "__rfloordiv__" or
-  name = "__rdiv__" or
-  name = "__rtruediv__" or
-  name = "__rmod__" or
-  name = "__rdivmod__" or
-  name = "__rpow__" or
-  name = "__rlshift__" or
-  name = "__rrshift__" or
-  name = "__rand__" or
-  name = "__rxor__" or
-  name = "__ror__" or
-  name = "__iadd__" or
-  name = "__isub__" or
-  name = "__imul__" or
-  name = "__ifloordiv__" or
-  name = "__idiv__" or
-  name = "__itruediv__" or
-  name = "__imod__" or
-  name = "__idivmod__" or
-  name = "__ipow__" or
-  name = "__ilshift__" or
-  name = "__irshift__" or
-  name = "__iand__" or
-  name = "__ixor__" or
-  name = "__ior__" or
-  name = "__coerce__"
+  name in [
+      "__lt__", "__le__", "__delattr__", "__delete__", "__instancecheck__", "__subclasscheck__",
+      "__getitem__", "__delitem__", "__contains__", "__add__", "__sub__", "__mul__", "__eq__",
+      "__floordiv__", "__div__", "__truediv__", "__mod__", "__divmod__", "__lshift__", "__rshift__",
+      "__and__", "__xor__", "__or__", "__ne__", "__radd__", "__rsub__", "__rmul__", "__rfloordiv__",
+      "__rdiv__", "__rtruediv__", "__rmod__", "__rdivmod__", "__rpow__", "__rlshift__", "__gt__",
+      "__rrshift__", "__rand__", "__rxor__", "__ror__", "__iadd__", "__isub__", "__imul__",
+      "__ifloordiv__", "__idiv__", "__itruediv__", "__ge__", "__imod__", "__idivmod__", "__ipow__",
+      "__ilshift__", "__irshift__", "__iand__", "__ixor__", "__ior__", "__coerce__", "__cmp__",
+      "__rcmp__", "__getattr___", "__getattribute___"
+    ]
 }
 
 predicate is_ternary_op(string name) {
-  name = "__setattr__" or
-  name = "__set__" or
-  name = "__setitem__" or
-  name = "__getslice__" or
-  name = "__delslice__"
+  name in ["__setattr__", "__set__", "__setitem__", "__getslice__", "__delslice__"]
 }
 
 predicate is_quad_op(string name) { name = "__setslice__" or name = "__exit__" }
@@ -132,12 +63,12 @@ predicate incorrect_special_method_defn(
     else
       if required < func.minParameters()
       then message = "Too many parameters" and show_counts = true
-      else
-        if func.minParameters() < required and not func.getScope().hasVarArg()
-        then
-          message = (required - func.minParameters()) + " default values(s) will never be used" and
-          show_counts = false
-        else none()
+      else (
+        func.minParameters() < required and
+        not func.getScope().hasVarArg() and
+        message = (required - func.minParameters()) + " default values(s) will never be used" and
+        show_counts = false
+      )
   )
 }
 

python/ql/src/Lexical/CommentedOutCode.qll

@@ -197,7 +197,7 @@ class CommentedOutCodeBlock extends @py_comment {
    * The location spans column `startcolumn` of line `startline` to
    * column `endcolumn` of line `endline` in file `filepath`.
    * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn
@@ -297,41 +297,17 @@ private predicate file_or_url(Comment c) {
   c.getText().regexpMatch("#[^'\"]+(\\[a-zA-Z]\\w*)+\\.[a-zA-Z]+.*")
 }
 
-private string operator_keyword() {
-  result = "import" or
-  result = "and" or
-  result = "is" or
-  result = "or" or
-  result = "in" or
-  result = "not" or
-  result = "as"
-}
+private string operator_keyword() { result in ["import", "and", "is", "or", "in", "not", "as"] }
 
 private string keyword_requiring_colon() {
-  result = "try" or
-  result = "while" or
-  result = "elif" or
-  result = "else" or
-  result = "if" or
-  result = "except" or
-  result = "def" or
-  result = "class"
+  result in ["try", "while", "elif", "else", "if", "except", "def", "class"]
 }
 
 private string other_keyword() {
-  result = "del" or
-  result = "lambda" or
-  result = "from" or
-  result = "global" or
-  result = "with" or
-  result = "assert" or
-  result = "yield" or
-  result = "finally" or
-  result = "print" or
-  result = "exec" or
-  result = "raise" or
-  result = "return" or
-  result = "for"
+  result in [
+      "del", "lambda", "raise", "return", "for", "from", "global", "with", "assert", "yield",
+      "finally", "print", "exec"
+    ]
 }
 
 private string a_keyword() {

python/ql/src/Metrics/Internal/Extents.qll

@@ -20,7 +20,7 @@ class RangeFunction extends Function {
    * The location spans column `startcolumn` of line `startline` to
    * column `endcolumn` of line `endline` in file `filepath`.
    * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn
@@ -40,7 +40,7 @@ class RangeClass extends Class {
    * The location spans column `startcolumn` of line `startline` to
    * column `endcolumn` of line `endline` in file `filepath`.
    * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn

python/ql/src/Security/CWE-089/SQLAlchemyTextClauseInjection.qhelp

@@ -1,53 +0,0 @@
-<!DOCTYPE qhelp PUBLIC
-  "-//Semmle//qhelp//EN"
-  "qhelp.dtd">
-<qhelp>
-
-<overview>
-<p>
-The <code>TextClause</code> class in the <code>SQLAlchemy</code> PyPI package represents
-a textual SQL string directly. If user-input is added to it without sufficient
-sanitization, a user may be able to run malicious database queries, since the
-<code>TextClause</code> is inserted directly into the final SQL.
-</p>
-</overview>
-
-<recommendation>
-<p>
-Don't allow user-input to be added to a <code>TextClause</code>, instead construct your
-full query with constructs from the ORM, or use query parameters for user-input.
-</p>
-</recommendation>
-
-<example>
-<p>
-In the following snippet, a user is fetched from the database using three
-different queries.
-</p>
-
-<p>
-In the first case, the final query string is built by directly including a user-supplied
-input. The parameter may include quote characters, so this code is vulnerable to a SQL
-injection attack.
-</p>
-
-<p>
-In the second case, the query is built using ORM models, but part of it is using a
-<code>TextClause</code> directly including a user-supplied input. Since the
-<code>TextClause</code> is inserted directly into the final SQL, this code is vulnerable
-to a SQL injection attack.
-</p>
-
-<p>
-In the third case, the query is built fully using the ORM models, so in the end, the
-user-supplied input will be passed to the database using query parameters. The
-database connector library will take care of escaping and inserting quotes as needed.
-</p>
-
-<sample src="examples/sqlalchemy_textclause_injection.py" />
-</example>
-
-<references>
-<li><a href="https://docs.sqlalchemy.org/en/14/core/sqlelement.html#sqlalchemy.sql.expression.text.params.text">Official documentation of the text parameter</a>.</li>
-</references>
-</qhelp>

python/ql/src/Security/CWE-089/SQLAlchemyTextClauseInjection.ql

@@ -1,23 +0,0 @@
-/**
- * @name SQLAlchemy TextClause built from user-controlled sources
- * @description Building a TextClause query from user-controlled sources is vulnerable to insertion of
- *              malicious SQL code by the user.
- * @kind path-problem
- * @problem.severity error
- * @security-severity 8.8
- * @precision high
- * @id py/sqlalchemy-textclause-injection
- * @tags security
- *       external/cwe/cwe-089
- *       external/owasp/owasp-a1
- */
-
-import python
-import semmle.python.security.dataflow.SQLAlchemyTextClause
-import DataFlow::PathGraph
-
-from SQLAlchemyTextClause::Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink
-where config.hasFlowPath(source, sink)
-select sink.getNode(), source, sink,
-  "This SQLAlchemy TextClause depends on $@, which could lead to SQL injection.", source.getNode(),
-  "a user-provided value"

python/ql/src/Security/CWE-089/SqlInjection.qhelp

@@ -9,6 +9,13 @@ If a database query (such as a SQL or NoSQL query) is built from
 user-provided data without sufficient sanitization, a user
 may be able to run malicious database queries.
 </p>
+
+<p>
+This also includes using the <code>TextClause</code> class in the
+<code><a href="https://pypi.org/project/SQLAlchemy/">SQLAlchemy</a></code> PyPI package,
+which is used to represent a literal SQL fragment and is inserted directly into the
+final SQL when used in a query built using the ORM.
+</p>
 </overview>
 
 <recommendation>
@@ -52,5 +59,6 @@ vulnerable to SQL injection attacks. In this example, if <code>username</code> w
 <references>
 <li>Wikipedia: <a href="https://en.wikipedia.org/wiki/SQL_injection">SQL injection</a>.</li>
 <li>OWASP: <a href="https://cheatsheetseries.owasp.org/cheatsheets/SQL_Injection_Prevention_Cheat_Sheet.html">SQL Injection Prevention Cheat Sheet</a>.</li>
+<li><a href="https://docs.sqlalchemy.org/en/14/core/sqlelement.html#sqlalchemy.sql.expression.text.params.text">SQLAlchemy documentation for TextClause</a>.</li>
 </references>
 </qhelp>

python/ql/src/Security/CWE-089/examples/sqlalchemy_textclause_injection.py

@@ -1,34 +0,0 @@
-from flask import Flask, request
-import sqlalchemy
-import sqlalchemy.orm
-
-app = Flask(__name__)
-engine = sqlalchemy.create_engine(...)
-Base = sqlalchemy.orm.declarative_base()
-
-
-class User(Base):
-    __tablename__ = "users"
-
-    id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
-    username = sqlalchemy.Column(sqlalchemy.String)
-
-
-@app.route("/users/<username>")
-def show_user(username):
-    session = sqlalchemy.orm.Session(engine)
-
-    # BAD, normal SQL injection
-    stmt = sqlalchemy.text("SELECT * FROM users WHERE username = '{}'".format(username))
-    results = session.execute(stmt).fetchall()
-
-    # BAD, allows SQL injection
-    username_formatted_for_sql = sqlalchemy.text("'{}'".format(username))
-    stmt = sqlalchemy.select(User).where(User.username == username_formatted_for_sql)
-    results = session.execute(stmt).scalars().all()
-
-    # GOOD, does not allow for SQL injection
-    stmt = sqlalchemy.select(User).where(User.username == username)
-    results = session.execute(stmt).scalars().all()
-
-    ...

python/ql/src/Security/CWE-730/RegexInjection.ql

@@ -5,25 +5,24 @@
  *              exponential time on certain inputs.
  * @kind path-problem
  * @problem.severity error
+ * @precision high
  * @id py/regex-injection
  * @tags security
  *       external/cwe/cwe-730
  *       external/cwe/cwe-400
  */
 
-// determine precision above
 import python
-import experimental.semmle.python.security.injection.RegexInjection
+private import semmle.python.Concepts
+import semmle.python.security.injection.RegexInjection
 import DataFlow::PathGraph
 
 from
-  RegexInjectionFlowConfig config, DataFlow::PathNode source, DataFlow::PathNode sink,
-  RegexInjectionSink regexInjectionSink, Attribute methodAttribute
+  RegexInjection::Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink,
+  RegexExecution regexExecution
 where
   config.hasFlowPath(source, sink) and
-  regexInjectionSink = sink.getNode() and
-  methodAttribute = regexInjectionSink.getRegexMethod()
+  regexExecution = sink.getNode().(RegexInjection::Sink).getRegexExecution()
 select sink.getNode(), source, sink,
   "$@ regular expression is constructed from a $@ and executed by $@.", sink.getNode(), "This",
-  source.getNode(), "user-provided value", methodAttribute,
-  regexInjectionSink.getRegexModule() + "." + methodAttribute.getName()
+  source.getNode(), "user-provided value", regexExecution, regexExecution.getName()

python/ql/src/Security/CWE-798/HardcodedCredentials.ql

@@ -88,7 +88,7 @@ class CredentialSink extends TaintSink {
   CredentialSink() {
     exists(string name |
       name.regexpMatch(getACredentialRegex()) and
-      not name.suffix(name.length() - 4) = "file"
+      not name.matches("%file")
     |
       any(FunctionValue func).getNamedArgumentForCall(_, name) = this
       or

python/ql/src/Statements/SideEffectInAssert.ql

@@ -15,16 +15,9 @@ import python
 
 predicate func_with_side_effects(Expr e) {
   exists(string name | name = e.(Attribute).getName() or name = e.(Name).getId() |
-    name = "print" or
-    name = "write" or
-    name = "append" or
-    name = "pop" or
-    name = "remove" or
-    name = "discard" or
-    name = "delete" or
-    name = "close" or
-    name = "open" or
-    name = "exit"
+    name in [
+        "print", "write", "append", "pop", "remove", "discard", "delete", "close", "open", "exit"
+      ]
   )
 }
 

python/ql/src/analysis/AlertSuppression.ql

@@ -88,7 +88,7 @@ class SuppressionScope extends @py_comment {
    * The location spans column `startcolumn` of line `startline` to
    * column `endcolumn` of line `endline` in file `filepath`.
    * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn

python/ql/src/analysis/Consistency.ql

@@ -8,15 +8,10 @@ import python
 import DefinitionTracking
 
 predicate uniqueness_error(int number, string what, string problem) {
-  (
-    what = "toString" or
-    what = "getLocation" or
-    what = "getNode" or
-    what = "getDefinition" or
-    what = "getEntryNode" or
-    what = "getOrigin" or
-    what = "getAnInferredType"
-  ) and
+  what in [
+      "toString", "getLocation", "getNode", "getDefinition", "getEntryNode", "getOrigin",
+      "getAnInferredType"
+    ] and
   (
     number = 0 and problem = "no results for " + what + "()"
     or
@@ -141,7 +136,7 @@ predicate builtin_object_consistency(string clsname, string problem, string what
     or
     not exists(o.toString()) and
     problem = "no toString" and
-    not exists(string name | name.prefix(7) = "_semmle" | py_special_objects(o, name)) and
+    not exists(string name | name.matches("\\_semmle%") | py_special_objects(o, name)) and
     not o = unknownValue()
   )
 }

python/ql/src/analysis/DefinitionTracking.qll

@@ -477,7 +477,7 @@ class NiceLocationExpr extends @py_expr {
    * The location spans column `bc` of line `bl` to
    * column `ec` of line `el` in file `f`.
    * For more information, see
-   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(string f, int bl, int bc, int el, int ec) {
     /* Attribute location for x.y is that of 'y' so that url does not overlap with that of 'x' */

python/ql/src/experimental/Security/CWE-113/HeaderInjection.qhelp

@@ -0,0 +1,26 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+<overview>
+<p>If an HTTP Header is built using string concatenation or string formatting, and the
+components of the concatenation include user input, a user 
+is likely to be able to manipulate the response.</p>
+</overview>
+
+<recommendation>
+<p>User input should not be included in an HTTP Header.</p>
+</recommendation>
+
+<example>
+<p>In the following example, the code appends a user-provided value into a header.</p>
+
+<sample src="header_injection.py" />
+</example>
+
+<references>
+<li>OWASP: <a href="https://owasp.org/www-community/attacks/HTTP_Response_Splitting">HTTP Response Splitting</a>.</li>
+<li>Python Security: <a href="https://python-security.readthedocs.io/vuln/http-header-injection.html">HTTP header injection</a>.</li>
+<li>SonarSource: <a href="https://rules.sonarsource.com/python/RSPEC-5167">RSPEC-5167</a>.</li>
+</references>
+</qhelp>

python/ql/src/experimental/Security/CWE-113/HeaderInjection.ql

@@ -0,0 +1,21 @@
+/**
+ * @name HTTP Header Injection
+ * @description User input should not be used in HTTP headers, otherwise a malicious user
+ *              may be able to inject a value that could manipulate the response.
+ * @kind path-problem
+ * @problem.severity error
+ * @id py/header-injection
+ * @tags security
+ *       external/cwe/cwe-113
+ *       external/cwe/cwe-079
+ */
+
+// determine precision above
+import python
+import experimental.semmle.python.security.injection.HTTPHeaders
+import DataFlow::PathGraph
+
+from HeaderInjectionFlowConfig config, DataFlow::PathNode source, DataFlow::PathNode sink
+where config.hasFlowPath(source, sink)
+select sink.getNode(), source, sink, "$@ HTTP header is constructed from a $@.", sink.getNode(),
+  "This", source.getNode(), "user-provided value"

python/ql/src/experimental/Security/CWE-113/header_injection.py

@@ -0,0 +1,9 @@
+from flask import Response, request, Flask, make_response
+
+
+@app.route("/flask_Response")
+def flask_Response():
+    rfs_header = request.args["rfs_header"]
+    response = Response()
+    response.headers['HeaderName'] = rfs_header
+    return response

python/ql/src/experimental/Security/CWE-117/LogInjection.qhelp

@@ -0,0 +1,49 @@
+
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<overview>
+
+<p>If unsanitized user input is written to a log entry, a malicious user may be able to forge new log entries.</p>
+
+<p>Forgery can occur if a user provides some input creating the appearance of multiple
+  log entries. This can include unescaped new-line characters, or HTML or other markup.</p>
+</overview>
+
+<recommendation>
+<p>
+User input should be suitably sanitized before it is logged.
+</p>
+<p>
+If the log entries are plain text then line breaks should be removed from user input, using for example
+<code>replace(old, new)</code> or similar. Care should also be taken that user input is clearly marked
+in log entries, and that a malicious user cannot cause confusion in other ways.
+</p>
+<p>
+For log entries that will be displayed in HTML, user input should be HTML encoded before being logged, to prevent forgery and
+other forms of HTML injection.
+</p>
+</recommendation>
+
+<example>
+<p>
+In the example, the name provided by the user is recorded using the log output function (<code>logging.info</code> or <code>app.logger.info</code>, etc.). 
+In these four cases, the name provided by the user is not provided The processing is recorded. If a malicious user provides <code>Guest%0D%0AUser name: Admin</code> 
+as a parameter, the log entry will be divided into two lines, the first line is <code>User name: Guest</code> code>, the second line is <code>User name: Admin</code>.
+</p>
+<sample src="LogInjectionBad.py" />
+
+<p>
+In a good example, the program uses the <code>replace</code> function to provide parameter processing to the user, and replace <code>\r\n</code> and <code>\n</code> 
+with empty characters. To a certain extent, the occurrence of log injection vulnerabilities is reduced.
+</p>
+
+<sample src="LogInjectionGood.py" />
+</example>
+
+<references>
+<li>OWASP: <a href="https://owasp.org/www-community/attacks/Log_Injection">Log Injection</a>.</li>
+</references>
+</qhelp>

python/ql/src/experimental/Security/CWE-117/LogInjection.ql

@@ -0,0 +1,20 @@
+/**
+ * @name Log Injection
+ * @description Building log entries from user-controlled data is vulnerable to
+ *              insertion of forged log entries by a malicious user.
+ * @kind path-problem
+ * @problem.severity error
+ * @precision high
+ * @id py/log-injection
+ * @tags security
+ *       external/cwe/cwe-117
+ */
+
+import python
+import experimental.semmle.python.security.injection.LogInjection
+import DataFlow::PathGraph
+
+from LogInjectionFlowConfig config, DataFlow::PathNode source, DataFlow::PathNode sink
+where config.hasFlowPath(source, sink)
+select sink.getNode(), source, sink, "$@ flows to log entry.", source.getNode(),
+  "User-provided value"

python/ql/src/experimental/Security/CWE-117/LogInjectionBad.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+@Desc   ：Log Injection
+"""
+from flask import Flask
+from flask import request
+from django.utils.log import request_logger
+import logging
+
+logging.basicConfig(level=logging.DEBUG)
+
+app = Flask(__name__)
+
+@app.route('/bad1')
+def bad1():
+    name = request.args.get('name')
+    app.logger.info('User name: ' + name) # Bad
+    return 'bad1'
+
+@app.route('/bad2')
+def bad2():
+    name = request.args.get('name')
+    logging.info('User name: ' + name) # Bad
+    return 'bad2'
+
+@app.route('/bad3')
+def bad3():
+    name = request.args.get('name')
+    request_logger.warn('User name: ' + name) # Bad
+    return 'bad3'
+
+@app.route('/bad4')
+def bad4():
+    name = request.args.get('name')
+    logtest = logging.getLogger('test')
+    logtest.debug('User name: ' + name) # Bad
+    return 'bad4'
+
+if __name__ == '__main__':
+    app.debug = True
+    handler = logging.FileHandler('log')
+    app.logger.addHandler(handler)
+    app.run()

python/ql/src/experimental/Security/CWE-117/LogInjectionGood.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+@Desc   ：Log Injection
+"""
+from flask import Flask
+from flask import request
+import logging
+
+logging.basicConfig(level=logging.DEBUG)
+
+app = Flask(__name__)
+
+@app.route('/good1')
+def good1():
+    name = request.args.get('name')
+    name = name.replace('\r\n','').replace('\n','')
+    logging.info('User name: ' + name) # Good
+    return 'good1'
+
+if __name__ == '__main__':
+    app.debug = True
+    handler = logging.FileHandler('log')
+    app.logger.addHandler(handler)
+    app.run()

python/ql/src/experimental/Security/CWE-348/ClientSuppliedIpUsedInSecurityCheck.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+@Desc   ：ip address spoofing
+"""
+from flask import Flask
+from flask import request
+
+app = Flask(__name__)
+
+@app.route('/bad1')
+def bad1():
+    client_ip = request.headers.get('x-forwarded-for')
+    if not client_ip.startswith('192.168.'):
+        raise Exception('ip illegal')
+    return 'bad1'
+
+@app.route('/bad2')
+def bad2():
+    client_ip = request.headers.get('x-forwarded-for')
+    if not client_ip == '127.0.0.1':
+        raise Exception('ip illegal')
+    return 'bad2'
+
+@app.route('/good1')
+def good1():
+    client_ip = request.headers.get('x-forwarded-for')
+    client_ip = client_ip.split(',')[client_ip.split(',').length - 1]
+    if not client_ip == '127.0.0.1':
+        raise Exception('ip illegal')
+    return 'good1'
+
+if __name__ == '__main__':
+    app.debug = True
+    app.run()

python/ql/src/experimental/Security/CWE-348/ClientSuppliedIpUsedInSecurityCheck.qhelp

@@ -0,0 +1,35 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+<overview>
+<p>An original client IP address is retrieved from an http header (<code>X-Forwarded-For</code> or <code>X-Real-IP</code> or <code>Proxy-Client-IP</code> 
+etc.), which is used to ensure security. Attackers can forge the value of these identifiers to
+bypass a ban-list, for example.</p>
+
+</overview>
+<recommendation>
+
+<p>Do not trust the values of HTTP headers allegedly identifying the originating IP. If you are aware your application will run behind some reverse proxies then the last entry of a <code>X-Forwarded-For</code> header value may be more trustworthy than the rest of it because some reverse proxies append the IP address they observed to the end of any remote-supplied header.</p>
+
+</recommendation>
+<example>
+
+<p>The following examples show the bad case and the good case respectively.
+In <code>bad1</code> method and <code>bad2</code> method, the client ip the <code>X-Forwarded-For</code> is split into comma-separated values, but the less-trustworthy first one is used. Both of these examples could be deceived by providing a forged HTTP header. The method
+<code>good1</code> similarly splits an <code>X-Forwarded-For</code> value, but uses the last, more-trustworthy entry.</p>
+
+<sample src="ClientSuppliedIpUsedInSecurityCheck.py" />
+
+</example>
+<references>
+
+<li>Dennis Schneider: <a href="https://www.dennis-schneider.com/blog/prevent-ip-address-spoofing-with-x-forwarded-for-header-and-aws-elb-in-clojure-ring/">
+Prevent IP address spoofing with X-Forwarded-For header when using AWS ELB and Clojure Ring</a>
+</li>
+
+<li>Security Rule Zero: <a href="https://www.f5.com/company/blog/security-rule-zero-a-warning-about-x-forwarded-for">A Warning about X-Forwarded-For</a>
+</li>
+
+</references>
+</qhelp>

python/ql/src/experimental/Security/CWE-348/ClientSuppliedIpUsedInSecurityCheck.ql

@@ -0,0 +1,56 @@
+/**
+ * @name IP address spoofing
+ * @description A remote endpoint identifier is read from an HTTP header. Attackers can modify the value
+ *              of the identifier to forge the client ip.
+ * @kind path-problem
+ * @problem.severity error
+ * @precision high
+ * @id py/ip-address-spoofing
+ * @tags security
+ *       external/cwe/cwe-348
+ */
+
+import python
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.dataflow.new.TaintTracking
+import semmle.python.ApiGraphs
+import ClientSuppliedIpUsedInSecurityCheckLib
+import DataFlow::PathGraph
+
+/**
+ * Taint-tracking configuration tracing flow from obtaining a client ip from an HTTP header to a sensitive use.
+ */
+class ClientSuppliedIpUsedInSecurityCheckConfig extends TaintTracking::Configuration {
+  ClientSuppliedIpUsedInSecurityCheckConfig() { this = "ClientSuppliedIpUsedInSecurityCheckConfig" }
+
+  override predicate isSource(DataFlow::Node source) {
+    source instanceof ClientSuppliedIpUsedInSecurityCheck
+  }
+
+  override predicate isSink(DataFlow::Node sink) { sink instanceof PossibleSecurityCheck }
+
+  override predicate isAdditionalTaintStep(DataFlow::Node pred, DataFlow::Node succ) {
+    exists(DataFlow::CallCfgNode ccn |
+      ccn = API::moduleImport("netaddr").getMember("IPAddress").getACall() and
+      ccn.getArg(0) = pred and
+      ccn = succ
+    )
+  }
+
+  override predicate isSanitizer(DataFlow::Node node) {
+    // `client_supplied_ip.split(",")[n]` for `n` > 0
+    exists(Subscript ss |
+      not ss.getIndex().(IntegerLiteral).getText() = "0" and
+      ss.getObject().(Call).getFunc().(Attribute).getName() = "split" and
+      ss.getObject().(Call).getAnArg().(StrConst).getText() = "," and
+      ss = node.asExpr()
+    )
+  }
+}
+
+from
+  ClientSuppliedIpUsedInSecurityCheckConfig config, DataFlow::PathNode source,
+  DataFlow::PathNode sink
+where config.hasFlowPath(source, sink)
+select sink.getNode(), source, sink, "IP address spoofing might include code from $@.",
+  source.getNode(), "this user input"

python/ql/src/experimental/Security/CWE-348/ClientSuppliedIpUsedInSecurityCheckLib.qll

@@ -0,0 +1,152 @@
+private import python
+private import semmle.python.Concepts
+private import semmle.python.ApiGraphs
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.dataflow.new.RemoteFlowSources
+
+/**
+ * A data flow source of the client ip obtained according to the remote endpoint identifier specified
+ * (`X-Forwarded-For`, `X-Real-IP`, `Proxy-Client-IP`, etc.) in the header.
+ *
+ * For example: `request.headers.get("X-Forwarded-For")`.
+ */
+abstract class ClientSuppliedIpUsedInSecurityCheck extends DataFlow::CallCfgNode { }
+
+private class FlaskClientSuppliedIpUsedInSecurityCheck extends ClientSuppliedIpUsedInSecurityCheck {
+  FlaskClientSuppliedIpUsedInSecurityCheck() {
+    exists(RemoteFlowSource rfs, DataFlow::AttrRead get |
+      rfs.getSourceType() = "flask.request" and this.getFunction() = get
+    |
+      // `get` is a call to request.headers.get or request.headers.get_all or request.headers.getlist
+      // request.headers
+      get.getObject()
+          .(DataFlow::AttrRead)
+          // request
+          .getObject()
+          .getALocalSource() = rfs and
+      get.getAttributeName() in ["get", "get_all", "getlist"] and
+      get.getObject().(DataFlow::AttrRead).getAttributeName() = "headers" and
+      this.getArg(0).asExpr().(StrConst).getText().toLowerCase() = clientIpParameterName()
+    )
+  }
+}
+
+private class DjangoClientSuppliedIpUsedInSecurityCheck extends ClientSuppliedIpUsedInSecurityCheck {
+  DjangoClientSuppliedIpUsedInSecurityCheck() {
+    exists(RemoteFlowSource rfs, DataFlow::AttrRead get |
+      rfs.getSourceType() = "django.http.request.HttpRequest" and this.getFunction() = get
+    |
+      // `get` is a call to request.headers.get or request.META.get
+      // request.headers
+      get.getObject()
+          .(DataFlow::AttrRead)
+          // request
+          .getObject()
+          .getALocalSource() = rfs and
+      get.getAttributeName() = "get" and
+      get.getObject().(DataFlow::AttrRead).getAttributeName() in ["headers", "META"] and
+      this.getArg(0).asExpr().(StrConst).getText().toLowerCase() = clientIpParameterName()
+    )
+  }
+}
+
+private class TornadoClientSuppliedIpUsedInSecurityCheck extends ClientSuppliedIpUsedInSecurityCheck {
+  TornadoClientSuppliedIpUsedInSecurityCheck() {
+    exists(RemoteFlowSource rfs, DataFlow::AttrRead get |
+      rfs.getSourceType() = "tornado.web.RequestHandler" and this.getFunction() = get
+    |
+      // `get` is a call to `rfs`.request.headers.get
+      // `rfs`.request.headers
+      get.getObject()
+          .(DataFlow::AttrRead)
+          // `rfs`.request
+          .getObject()
+          .(DataFlow::AttrRead)
+          // `rfs`
+          .getObject()
+          .getALocalSource() = rfs and
+      get.getAttributeName() in ["get", "get_list"] and
+      get.getObject().(DataFlow::AttrRead).getAttributeName() = "headers" and
+      this.getArg(0).asExpr().(StrConst).getText().toLowerCase() = clientIpParameterName()
+    )
+  }
+}
+
+private string clientIpParameterName() {
+  result in [
+      "x-forwarded-for", "x_forwarded_for", "x-real-ip", "x_real_ip", "proxy-client-ip",
+      "proxy_client_ip", "wl-proxy-client-ip", "wl_proxy_client_ip", "http_x_forwarded_for",
+      "http-x-forwarded-for", "http_x_forwarded", "http_x_cluster_client_ip", "http_client_ip",
+      "http_forwarded_for", "http_forwarded", "http_via", "remote_addr"
+    ]
+}
+
+/** A data flow sink for ip address forgery vulnerabilities. */
+abstract class PossibleSecurityCheck extends DataFlow::Node { }
+
+/** A data flow sink for sql operation. */
+private class SqlOperationAsSecurityCheck extends PossibleSecurityCheck {
+  SqlOperationAsSecurityCheck() { this = any(SqlExecution e).getSql() }
+}
+
+/**
+ * A data flow sink for remote client ip comparison.
+ *
+ * For example: `if not ipAddr.startswith('192.168.') : ...` determine whether the client ip starts
+ * with `192.168.`, and the program can be deceived by forging the ip address.
+ */
+private class CompareSink extends PossibleSecurityCheck {
+  CompareSink() {
+    exists(Call call |
+      call.getFunc().(Attribute).getName() = "startswith" and
+      call.getArg(0).(StrConst).getText().regexpMatch(getIpAddressRegex()) and
+      not call.getArg(0).(StrConst).getText() = "0:0:0:0:0:0:0:1" and
+      call.getFunc().(Attribute).getObject() = this.asExpr()
+    )
+    or
+    exists(Compare compare |
+      (
+        compare.getOp(0) instanceof Eq or
+        compare.getOp(0) instanceof NotEq
+      ) and
+      (
+        compare.getLeft() = this.asExpr() and
+        compare.getComparator(0).(StrConst).getText() instanceof PrivateHostName and
+        not compare.getComparator(0).(StrConst).getText() = "0:0:0:0:0:0:0:1"
+        or
+        compare.getComparator(0) = this.asExpr() and
+        compare.getLeft().(StrConst).getText() instanceof PrivateHostName and
+        not compare.getLeft().(StrConst).getText() = "0:0:0:0:0:0:0:1"
+      )
+    )
+    or
+    exists(Compare compare |
+      (
+        compare.getOp(0) instanceof In or
+        compare.getOp(0) instanceof NotIn
+      ) and
+      (
+        compare.getLeft() = this.asExpr()
+        or
+        compare.getComparator(0) = this.asExpr() and
+        not compare.getLeft().(StrConst).getText() in ["%", ",", "."]
+      )
+    )
+  }
+}
+
+string getIpAddressRegex() {
+  result =
+    "^((10\\.((1\\d{2})?|(2[0-4]\\d)?|(25[0-5])?|([1-9]\\d|[0-9])?)(\\.)?)|(192\\.168\\.)|172\\.(1[6789]|2[0-9]|3[01])\\.)((1\\d{2})?|(2[0-4]\\d)?|(25[0-5])?|([1-9]\\d|[0-9])?)(\\.)?((1\\d{2})?|(2[0-4]\\d)?|(25[0-5])?|([1-9]\\d|[0-9])?)$"
+}
+
+/**
+ * A string matching private host names of IPv4 and IPv6, which only matches the host portion therefore checking for port is not necessary.
+ * Several examples are localhost, reserved IPv4 IP addresses including 127.0.0.1, 10.x.x.x, 172.16.x,x, 192.168.x,x, and reserved IPv6 addresses including [0:0:0:0:0:0:0:1] and [::1]
+ */
+private class PrivateHostName extends string {
+  bindingset[this]
+  PrivateHostName() {
+    this.regexpMatch("(?i)localhost(?:[:/?#].*)?|127\\.0\\.0\\.1(?:[:/?#].*)?|10(?:\\.[0-9]+){3}(?:[:/?#].*)?|172\\.16(?:\\.[0-9]+){2}(?:[:/?#].*)?|192.168(?:\\.[0-9]+){2}(?:[:/?#].*)?|\\[?0:0:0:0:0:0:0:1\\]?(?:[:/?#].*)?|\\[?::1\\]?(?:[:/?#].*)?")
+  }
+}

python/ql/src/experimental/Security/CWE-522/LDAPInsecureAuth.qhelp

@@ -0,0 +1,23 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<overview>
+<p>Failing to ensure the utilization of SSL in an LDAP connection can cause the entire communication
+to be sent in cleartext making it easier for an attacker to intercept it.</p>
+</overview>
+
+<recommendation>
+<p>Always set <code>use_SSL</code> to <code>True</code>, call <code>start_tls_s()</code> or set a proper option flag (<code>ldap.OPT_X_TLS_XXXXXX</code>).</p>
+</recommendation>
+
+<example>
+<p>This example shows both good and bad ways to deal with this issue under Python 3.</p>
+
+<p>The first one sets <code>use_SSL</code> to true as a keyword argument whereas the second one fails to provide a value for it, so
+the default one is used (<code>False</code>).</p>
+<sample src="examples/LDAPInsecureAuth.py" />
+</example>
+
+</qhelp>

python/ql/src/experimental/Security/CWE-522/LDAPInsecureAuth.ql

@@ -0,0 +1,21 @@
+/**
+ * @name Python Insecure LDAP Authentication
+ * @description Python LDAP Insecure LDAP Authentication
+ * @kind path-problem
+ * @problem.severity error
+ * @id py/insecure-ldap-auth
+ * @tags experimental
+ *       security
+ *       external/cwe/cwe-522
+ *       external/cwe/cwe-523
+ */
+
+// determine precision above
+import python
+import DataFlow::PathGraph
+import experimental.semmle.python.security.LDAPInsecureAuth
+
+from LDAPInsecureAuthConfig config, DataFlow::PathNode source, DataFlow::PathNode sink
+where config.hasFlowPath(source, sink)
+select sink.getNode(), source, sink, "$@ is authenticated insecurely.", sink.getNode(),
+  "This LDAP host"