Merge pull request #1 from github/main

Sync with the upstream

python/ql/lib/CHANGELOG.md

@@ -1,3 +1,35 @@
+## 0.7.1
+
+No user-facing changes.
+
+## 0.7.0
+
+### Major Analysis Improvements
+
+* The _PAM authorization bypass due to incorrect usage_ (`py/pam-auth-bypass`) query has been converted to a taint-tracking query, resulting in significantly fewer false positives.
+
+### Minor Analysis Improvements
+
+* Added `subprocess.getoutput` and `subprocess.getoutputstatus` as new command injection sinks for the StdLib.
+* The data-flow library has been rewritten to no longer rely on the points-to analysis in order to resolve references to modules. Improvements in the module resolution can lead to more results.
+* Deleted the deprecated `importNode` predicate from the `DataFlowUtil.qll` file.
+* Deleted the deprecated features from `PEP249.qll` that were not inside the `PEP249` module.
+* Deleted the deprecated `werkzeug` from the `Werkzeug` module in `Werkzeug.qll`.
+* Deleted the deprecated `methodResult` predicate from `PEP249::Cursor`.
+
+### Bug Fixes
+
+* `except*` is now supported.
+* The result of `Try.getAHandler` and `Try.getHandler(<index>)` is no longer of type `ExceptStmt`, as handlers may also be `ExceptGroupStmt`s (After Python 3.11 introduced PEP 654). Instead, it is of the new type `ExceptionHandler` of which `ExceptStmt` and `ExceptGroupStmt` are subtypes. To support selecting only one type of handler, `Try.getANormalHandler` and `Try.getAGroupHandler` have been added. Existing uses of `Try.getAHandler` for which it is important to select only normal handlers, will need to be updated to `Try.getANormalHandler`.
+
+## 0.6.6
+
+No user-facing changes.
+
+## 0.6.5
+
+No user-facing changes.
+
 ## 0.6.4
 
 ### Minor Analysis Improvements

python/ql/lib/analysis/DefinitionTracking.qll

@@ -221,10 +221,8 @@ pragma[noinline]
 private predicate module_and_name_for_import_star(
   ModuleObject mod, string name, ImportStarRefinement def
 ) {
-  exists(ImportStarNode im_star |
-    module_and_name_for_import_star_helper(mod, name, im_star, def) and
-    mod.exports(name)
-  )
+  module_and_name_for_import_star_helper(mod, name, _, def) and
+  mod.exports(name)
 }
 
 pragma[noinline]
@@ -484,9 +482,9 @@ class NiceLocationExpr extends Expr {
    */
   predicate hasLocationInfo(string f, int bl, int bc, int el, int ec) {
     /* Attribute location for x.y is that of 'y' so that url does not overlap with that of 'x' */
-    exists(int abl, int abc | this.(Attribute).getLocation().hasLocationInfo(f, abl, abc, el, ec) |
-      bl = el and bc = ec - this.(Attribute).getName().length() + 1
-    )
+    this.(Attribute).getLocation().hasLocationInfo(f, _, _, el, ec) and
+    bl = el and
+    bc = ec - this.(Attribute).getName().length() + 1
     or
     this.(Name).getLocation().hasLocationInfo(f, bl, bc, el, ec)
     or

python/ql/lib/change-notes/2022-11-17-deleted-deps.md

@@ -1,7 +0,0 @@
----
-category: minorAnalysis
----
-* Deleted the deprecated `importNode` predicate from the `DataFlowUtil.qll` file.
-* Deleted the deprecated features from `PEP249.qll` that were not inside the `PEP249` module.
-* Deleted the deprecated `werkzeug` from the `Werkzeug` module in `Werkzeug.qll`.
-* Deleted the deprecated `methodResult` predicate from `PEP249::Cursor`.

python/ql/lib/change-notes/2022-11-21-module-resolution-rewrite.md

@@ -1,5 +0,0 @@
----
- category: minorAnalysis
----
- * The data-flow library has been rewritten to no longer rely on the points-to analysis in order to
-   resolve references to modules. Improvements in the module resolution can lead to more results.

python/ql/lib/change-notes/released/0.6.5.md

@@ -0,0 +1,3 @@
+## 0.6.5
+
+No user-facing changes.

python/ql/lib/change-notes/released/0.6.6.md

@@ -0,0 +1,3 @@
+## 0.6.6
+
+No user-facing changes.

python/ql/lib/change-notes/released/0.7.0.md

@@ -0,0 +1,19 @@
+## 0.7.0
+
+### Major Analysis Improvements
+
+* The _PAM authorization bypass due to incorrect usage_ (`py/pam-auth-bypass`) query has been converted to a taint-tracking query, resulting in significantly fewer false positives.
+
+### Minor Analysis Improvements
+
+* Added `subprocess.getoutput` and `subprocess.getoutputstatus` as new command injection sinks for the StdLib.
+* The data-flow library has been rewritten to no longer rely on the points-to analysis in order to resolve references to modules. Improvements in the module resolution can lead to more results.
+* Deleted the deprecated `importNode` predicate from the `DataFlowUtil.qll` file.
+* Deleted the deprecated features from `PEP249.qll` that were not inside the `PEP249` module.
+* Deleted the deprecated `werkzeug` from the `Werkzeug` module in `Werkzeug.qll`.
+* Deleted the deprecated `methodResult` predicate from `PEP249::Cursor`.
+
+### Bug Fixes
+
+* `except*` is now supported.
+* The result of `Try.getAHandler` and `Try.getHandler(<index>)` is no longer of type `ExceptStmt`, as handlers may also be `ExceptGroupStmt`s (After Python 3.11 introduced PEP 654). Instead, it is of the new type `ExceptionHandler` of which `ExceptStmt` and `ExceptGroupStmt` are subtypes. To support selecting only one type of handler, `Try.getANormalHandler` and `Try.getAGroupHandler` have been added. Existing uses of `Try.getAHandler` for which it is important to select only normal handlers, will need to be updated to `Try.getANormalHandler`.

python/ql/lib/change-notes/released/0.7.1.md

@@ -0,0 +1,3 @@
+## 0.7.1
+
+No user-facing changes.

python/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.6.4
+lastReleaseVersion: 0.7.1

python/ql/lib/qlpack.yml

@@ -1,9 +1,12 @@
 name: codeql/python-all
-version: 0.6.5-dev
+version: 0.7.2-dev
 groups: python
 dbscheme: semmlecode.python.dbscheme
 extractor: python
 library: true
 upgrades: upgrades
 dependencies:
-  codeql/regex: ${workspace}
+  codeql/regex: ${workspace}
+  codeql/tutorial: ${workspace}
+dataExtensions:
+  - semmle/python/frameworks/**/model.yml

python/ql/lib/semmle/python/AstGenerated.qll

@@ -389,6 +389,26 @@ class Eq_ extends @py_Eq, Cmpop {
   override string toString() { result = "Eq" }
 }
 
+/** INTERNAL: See the class `ExceptGroupStmt` for further information. */
+class ExceptGroupStmt_ extends @py_ExceptGroupStmt, Stmt {
+  /** Gets the type of this except group block. */
+  Expr getType() { py_exprs(result, _, this, 1) }
+
+  /** Gets the name of this except group block. */
+  Expr getName() { py_exprs(result, _, this, 2) }
+
+  /** Gets the body of this except group block. */
+  StmtList getBody() { py_stmt_lists(result, this, 3) }
+
+  /** Gets the nth statement of this except group block. */
+  Stmt getStmt(int index) { result = this.getBody().getItem(index) }
+
+  /** Gets a statement of this except group block. */
+  Stmt getAStmt() { result = this.getBody().getAnItem() }
+
+  override string toString() { result = "ExceptGroupStmt" }
+}
+
 /** INTERNAL: See the class `ExceptStmt` for further information. */
 class ExceptStmt_ extends @py_ExceptStmt, Stmt {
   /** Gets the type of this except block. */

python/ql/lib/semmle/python/Concepts.qll

@@ -9,6 +9,7 @@ private import semmle.python.dataflow.new.DataFlow
 private import semmle.python.dataflow.new.RemoteFlowSources
 private import semmle.python.dataflow.new.TaintTracking
 private import semmle.python.Frameworks
+private import semmle.python.security.internal.EncryptionKeySizes
 
 /**
  * A data-flow node that executes an operating system command,
@@ -1141,21 +1142,21 @@ module Cryptography {
       abstract class RsaRange extends Range {
         final override string getName() { result = "RSA" }
 
-        final override int minimumSecureKeySize() { result = 2048 }
+        final override int minimumSecureKeySize() { result = minSecureKeySizeRsa() }
       }
 
       /** A data-flow node that generates a new DSA key-pair. */
       abstract class DsaRange extends Range {
         final override string getName() { result = "DSA" }
 
-        final override int minimumSecureKeySize() { result = 2048 }
+        final override int minimumSecureKeySize() { result = minSecureKeySizeDsa() }
       }
 
       /** A data-flow node that generates a new ECC key-pair. */
       abstract class EccRange extends Range {
         final override string getName() { result = "ECC" }
 
-        final override int minimumSecureKeySize() { result = 224 }
+        final override int minimumSecureKeySize() { result = minSecureKeySizeEcc() }
       }
     }
   }

python/ql/lib/semmle/python/Flow.qll

@@ -385,9 +385,9 @@ class CallNode extends ControlFlowNode {
 
   /** Gets the flow node corresponding to an argument of the call corresponding to this flow node */
   ControlFlowNode getAnArg() {
-    exists(int n | result = this.getArg(n))
+    result = this.getArg(_)
     or
-    exists(string name | result = this.getArgByName(name))
+    result = this.getArgByName(_)
   }
 
   override Call getNode() { result = super.getNode() }

python/ql/lib/semmle/python/RegexTreeView.qll

@@ -102,6 +102,9 @@ module Impl implements RegexTreeViewSig {
     /** Gets the number of child terms. */
     int getNumChild() { result = count(this.getAChild()) }
 
+    /** Gets the last child term of this element. */
+    RegExpTerm getLastChild() { result = this.getChild(this.getNumChild() - 1) }
+
     /** Gets the associated regex. */
     abstract Regex getRegex();
   }
@@ -224,8 +227,8 @@ module Impl implements RegexTreeViewSig {
     predicate hasLocationInfo(
       string filepath, int startline, int startcolumn, int endline, int endcolumn
     ) {
-      exists(int re_start, int re_end |
-        re.getLocation().hasLocationInfo(filepath, startline, re_start, endline, re_end) and
+      exists(int re_start |
+        re.getLocation().hasLocationInfo(filepath, startline, re_start, endline, _) and
         startcolumn = re_start + start + 4 and
         endcolumn = re_start + end + 3
       )
@@ -454,7 +457,16 @@ module Impl implements RegexTreeViewSig {
     override string getPrimaryQLClass() { result = "RegExpAlt" }
   }
 
-  additional class RegExpCharEscape = RegExpEscape;
+  /**
+   * A character escape in a regular expression.
+   *
+   * Example:
+   *
+   * ```
+   * \.
+   * ```
+   */
+  class RegExpCharEscape = RegExpEscape;
 
   /**
    * An escaped regular expression term, that is, a regular expression
@@ -561,6 +573,13 @@ module Impl implements RegexTreeViewSig {
     RegExpWordBoundary() { this.getChar() = "\\b" }
   }
 
+  /**
+   * A non-word boundary, that is, a regular expression term of the form `\B`.
+   */
+  class RegExpNonWordBoundary extends RegExpSpecialChar {
+    RegExpNonWordBoundary() { this.getChar() = "\\B" }
+  }
+
   /**
    * A character class escape in a regular expression.
    * That is, an escaped character that denotes multiple characters.
@@ -829,6 +848,19 @@ module Impl implements RegexTreeViewSig {
     override string getPrimaryQLClass() { result = "RegExpDot" }
   }
 
+  /**
+   * A term that matches a specific position between characters in the string.
+   *
+   * Example:
+   *
+   * ```
+   * \A
+   * ```
+   */
+  class RegExpAnchor extends RegExpSpecialChar {
+    RegExpAnchor() { this.getChar() = ["\\A", "^", "$", "\\Z"] }
+  }
+
   /**
    * A dollar assertion `$` or `\Z` matching the end of a line.
    *
@@ -838,7 +870,7 @@ module Impl implements RegexTreeViewSig {
    * $
    * ```
    */
-  class RegExpDollar extends RegExpSpecialChar {
+  class RegExpDollar extends RegExpAnchor {
     RegExpDollar() { this.getChar() = ["$", "\\Z"] }
 
     override string getPrimaryQLClass() { result = "RegExpDollar" }
@@ -853,7 +885,7 @@ module Impl implements RegexTreeViewSig {
    * ^
    * ```
    */
-  class RegExpCaret extends RegExpSpecialChar {
+  class RegExpCaret extends RegExpAnchor {
     RegExpCaret() { this.getChar() = ["^", "\\A"] }
 
     override string getPrimaryQLClass() { result = "RegExpCaret" }

python/ql/lib/semmle/python/SelfAttribute.qll

@@ -78,8 +78,8 @@ private predicate attr_assigned_in_method_arg_n(FunctionObject method, string na
       attr.isStore()
     )
     or
-    exists(CallNode call, FunctionObject callee, int m |
-      callee.getArgumentForCall(call, m) = param.getAUse() and
+    exists(FunctionObject callee, int m |
+      callee.getArgumentForCall(_, m) = param.getAUse() and
       attr_assigned_in_method_arg_n(callee, name, m)
     )
   )

python/ql/lib/semmle/python/SpecialMethods.qll

@@ -11,9 +11,7 @@
 private import python
 
 /** A control flow node which might correspond to a special method call. */
-class PotentialSpecialMethodCallNode extends ControlFlowNode {
-  PotentialSpecialMethodCallNode() { this instanceof SpecialMethod::Potential }
-}
+class PotentialSpecialMethodCallNode extends ControlFlowNode instanceof SpecialMethod::Potential { }
 
 /**
  * Machinery for detecting special method calls.

python/ql/lib/semmle/python/Stmts.qll

@@ -143,12 +143,30 @@ class Exec extends Exec_ {
   override Stmt getASubStatement() { none() }
 }
 
-/** An except statement (part of a `try` statement), such as `except IOError as err:` */
-class ExceptStmt extends ExceptStmt_ {
-  /* syntax: except Expr [ as Expr ]: */
+/**
+ * An exception handler such as an `except` or an `except*` statement
+ * in a `try` statement.
+ */
+class ExceptionHandler extends Stmt {
+  ExceptionHandler() {
+    this instanceof ExceptStmt_
+    or
+    this instanceof ExceptGroupStmt_
+  }
+
   /** Gets the immediately enclosing try statement */
   Try getTry() { result.getAHandler() = this }
 
+  /** Gets the name of this except group block. */
+  abstract Expr getName();
+
+  /** Gets the type of this except group block. */
+  abstract Expr getType();
+}
+
+/** An except group statement (part of a `try` statement), such as `except* IOError as err:` */
+class ExceptGroupStmt extends ExceptGroupStmt_, ExceptionHandler {
+  /* syntax: except Expr [ as Expr ]: */
   override Expr getASubExpression() {
     result = this.getName()
     or
@@ -159,10 +177,34 @@ class ExceptStmt extends ExceptStmt_ {
 
   override Stmt getLastStatement() { result = this.getBody().getLastItem().getLastStatement() }
 
+  override Expr getName() { result = ExceptGroupStmt_.super.getName() }
+
   override Expr getType() {
-    result = super.getType() and not result instanceof Tuple
+    result = ExceptGroupStmt_.super.getType() and not result instanceof Tuple
     or
-    result = super.getType().(Tuple).getAnElt()
+    result = ExceptGroupStmt_.super.getType().(Tuple).getAnElt()
+  }
+}
+
+/** An except statement (part of a `try` statement), such as `except IOError as err:` */
+class ExceptStmt extends ExceptStmt_, ExceptionHandler {
+  /* syntax: except Expr [ as Expr ]: */
+  override Expr getASubExpression() {
+    result = this.getName()
+    or
+    result = this.getType()
+  }
+
+  override Stmt getASubStatement() { result = this.getAStmt() }
+
+  override Stmt getLastStatement() { result = this.getBody().getLastItem().getLastStatement() }
+
+  override Expr getName() { result = ExceptStmt_.super.getName() }
+
+  override Expr getType() {
+    result = ExceptStmt_.super.getType() and not result instanceof Tuple
+    or
+    result = ExceptStmt_.super.getType().(Tuple).getAnElt()
   }
 }
 
@@ -364,10 +406,15 @@ class Try extends Try_ {
     result = this.getAnOrelse()
   }
 
-  override ExceptStmt getHandler(int i) { result = Try_.super.getHandler(i) }
+  override ExceptionHandler getHandler(int i) { result = Try_.super.getHandler(i) }
 
-  /** Gets an exception handler of this try statement. */
-  override ExceptStmt getAHandler() { result = Try_.super.getAHandler() }
+  override ExceptionHandler getAHandler() { result = Try_.super.getAHandler() }
+
+  /** Gets a normal exception handler, `except`, of this try statement. */
+  ExceptStmt getANormalHandler() { result = this.getAHandler() }
+
+  /** Gets a group exception handler, `except*`, of this try statement. */
+  ExceptGroupStmt getAGroupHandler() { result = this.getAHandler() }
 
   override Stmt getLastStatement() {
     result = this.getFinalbody().getLastItem().getLastStatement()

python/ql/lib/semmle/python/dataflow/new/Regexp.qll

@@ -9,11 +9,12 @@ private import semmle.python.dataflow.new.DataFlow
 /**
  * Provides utility predicates related to regular expressions.
  */
-module RegExpPatterns {
+deprecated module RegExpPatterns {
   /**
    * Gets a pattern that matches common top-level domain names in lower case.
+   * DEPRECATED: use the similarly named predicate from `HostnameRegex` from the `regex` pack instead.
    */
-  string getACommonTld() {
+  deprecated string getACommonTld() {
     // according to ranking by http://google.com/search?q=site:.<<TLD>>
     result = "(?:com|org|edu|gov|uk|net|io)(?![a-z0-9])"
   }

python/ql/lib/semmle/python/dataflow/new/RemoteFlowSources.qll

@@ -15,13 +15,9 @@ private import semmle.python.Concepts
  * Extend this class to refine existing API models. If you want to model new APIs,
  * extend `RemoteFlowSource::Range` instead.
  */
-class RemoteFlowSource extends DataFlow::Node {
-  RemoteFlowSource::Range self;
-
-  RemoteFlowSource() { this = self }
-
+class RemoteFlowSource extends DataFlow::Node instanceof RemoteFlowSource::Range {
   /** Gets a string that describes the type of this remote flow source. */
-  string getSourceType() { result = self.getSourceType() }
+  string getSourceType() { result = super.getSourceType() }
 }
 
 /** Provides a class for modeling new sources of remote user input. */

python/ql/lib/semmle/python/dataflow/new/SensitiveDataSources.qll

@@ -21,11 +21,8 @@ module SensitiveDataClassification = SensitiveDataHeuristics::SensitiveDataClass
  * Extend this class to refine existing API models. If you want to model new APIs,
  * extend `SensitiveDataSource::Range` instead.
  */
-class SensitiveDataSource extends DataFlow::Node {
-  SensitiveDataSource::Range range;
-
+class SensitiveDataSource extends DataFlow::Node instanceof SensitiveDataSource::Range {
   SensitiveDataSource() {
-    this = range and
     // ignore sensitive password sources in getpass.py, that can escape through `getpass.getpass()` return value,
     // since `getpass.getpass()` is considered a source itself.
     not exists(Module getpass |
@@ -39,7 +36,7 @@ class SensitiveDataSource extends DataFlow::Node {
   /**
    * Gets the classification of the sensitive data.
    */
-  SensitiveDataClassification getClassification() { result = range.getClassification() }
+  SensitiveDataClassification getClassification() { result = super.getClassification() }
 }
 
 /** Provides a class for modeling new sources of sensitive data, such as secrets, certificates, or passwords. */

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplCommon.qll

@@ -916,28 +916,56 @@ private module Cached {
     TDataFlowCallSome(DataFlowCall call)
 
   cached
-  newtype TParameterPositionOption =
-    TParameterPositionNone() or
-    TParameterPositionSome(ParameterPosition pos)
+  newtype TParamNodeOption =
+    TParamNodeNone() or
+    TParamNodeSome(ParamNode p)
 
   cached
   newtype TReturnCtx =
     TReturnCtxNone() or
     TReturnCtxNoFlowThrough() or
-    TReturnCtxMaybeFlowThrough(ReturnKindExt kind)
+    TReturnCtxMaybeFlowThrough(ReturnPosition pos)
+
+  cached
+  newtype TTypedContentApprox =
+    MkTypedContentApprox(ContentApprox c, DataFlowType t) {
+      exists(Content cont |
+        c = getContentApprox(cont) and
+        store(_, cont, _, _, t)
+      )
+    }
 
   cached
   newtype TTypedContent = MkTypedContent(Content c, DataFlowType t) { store(_, c, _, _, t) }
 
+  cached
+  TypedContent getATypedContent(TypedContentApprox c) {
+    exists(ContentApprox cls, DataFlowType t, Content cont |
+      c = MkTypedContentApprox(cls, pragma[only_bind_into](t)) and
+      result = MkTypedContent(cont, pragma[only_bind_into](t)) and
+      cls = getContentApprox(cont)
+    )
+  }
+
   cached
   newtype TAccessPathFront =
     TFrontNil(DataFlowType t) or
     TFrontHead(TypedContent tc)
 
+  cached
+  newtype TApproxAccessPathFront =
+    TApproxFrontNil(DataFlowType t) or
+    TApproxFrontHead(TypedContentApprox tc)
+
   cached
   newtype TAccessPathFrontOption =
     TAccessPathFrontNone() or
     TAccessPathFrontSome(AccessPathFront apf)
+
+  cached
+  newtype TApproxAccessPathFrontOption =
+    TApproxAccessPathFrontNone() or
+    TApproxAccessPathFrontSome(ApproxAccessPathFront apf)
 }
 
 /**
@@ -1315,15 +1343,15 @@ class DataFlowCallOption extends TDataFlowCallOption {
   }
 }
 
-/** An optional `ParameterPosition`. */
-class ParameterPositionOption extends TParameterPositionOption {
+/** An optional `ParamNode`. */
+class ParamNodeOption extends TParamNodeOption {
   string toString() {
-    this = TParameterPositionNone() and
+    this = TParamNodeNone() and
     result = "(none)"
     or
-    exists(ParameterPosition pos |
-      this = TParameterPositionSome(pos) and
-      result = pos.toString()
+    exists(ParamNode p |
+      this = TParamNodeSome(p) and
+      result = p.toString()
     )
   }
 }
@@ -1335,7 +1363,7 @@ class ParameterPositionOption extends TParameterPositionOption {
  *
  * - `TReturnCtxNone()`: no return flow.
  * - `TReturnCtxNoFlowThrough()`: return flow, but flow through is not possible.
- * - `TReturnCtxMaybeFlowThrough(ReturnKindExt kind)`: return flow, of kind `kind`, and
+ * - `TReturnCtxMaybeFlowThrough(ReturnPosition pos)`: return flow, of kind `pos`, and
  *    flow through may be possible.
  */
 class ReturnCtx extends TReturnCtx {
@@ -1346,13 +1374,82 @@ class ReturnCtx extends TReturnCtx {
     this = TReturnCtxNoFlowThrough() and
     result = "(no flow through)"
     or
-    exists(ReturnKindExt kind |
-      this = TReturnCtxMaybeFlowThrough(kind) and
-      result = kind.toString()
+    exists(ReturnPosition pos |
+      this = TReturnCtxMaybeFlowThrough(pos) and
+      result = pos.toString()
     )
   }
 }
 
+/** An approximated `Content` tagged with the type of a containing object. */
+class TypedContentApprox extends MkTypedContentApprox {
+  private ContentApprox c;
+  private DataFlowType t;
+
+  TypedContentApprox() { this = MkTypedContentApprox(c, t) }
+
+  /** Gets a typed content approximated by this value. */
+  TypedContent getATypedContent() { result = getATypedContent(this) }
+
+  /** Gets the container type. */
+  DataFlowType getContainerType() { result = t }
+
+  /** Gets a textual representation of this approximated content. */
+  string toString() { result = c.toString() }
+}
+
+/**
+ * The front of an approximated access path. This is either a head or a nil.
+ */
+abstract class ApproxAccessPathFront extends TApproxAccessPathFront {
+  abstract string toString();
+
+  abstract DataFlowType getType();
+
+  abstract boolean toBoolNonEmpty();
+
+  pragma[nomagic]
+  TypedContent getAHead() {
+    exists(TypedContentApprox cont |
+      this = TApproxFrontHead(cont) and
+      result = cont.getATypedContent()
+    )
+  }
+}
+
+class ApproxAccessPathFrontNil extends ApproxAccessPathFront, TApproxFrontNil {
+  private DataFlowType t;
+
+  ApproxAccessPathFrontNil() { this = TApproxFrontNil(t) }
+
+  override string toString() { result = ppReprType(t) }
+
+  override DataFlowType getType() { result = t }
+
+  override boolean toBoolNonEmpty() { result = false }
+}
+
+class ApproxAccessPathFrontHead extends ApproxAccessPathFront, TApproxFrontHead {
+  private TypedContentApprox tc;
+
+  ApproxAccessPathFrontHead() { this = TApproxFrontHead(tc) }
+
+  override string toString() { result = tc.toString() }
+
+  override DataFlowType getType() { result = tc.getContainerType() }
+
+  override boolean toBoolNonEmpty() { result = true }
+}
+
+/** An optional approximated access path front. */
+class ApproxAccessPathFrontOption extends TApproxAccessPathFrontOption {
+  string toString() {
+    this = TApproxAccessPathFrontNone() and result = "<none>"
+    or
+    this = TApproxAccessPathFrontSome(any(ApproxAccessPathFront apf | result = apf.toString()))
+  }
+}
+
 /** A `Content` tagged with the type of a containing object. */
 class TypedContent extends MkTypedContent {
   private Content c;
@@ -1385,7 +1482,7 @@ abstract class AccessPathFront extends TAccessPathFront {
 
   abstract DataFlowType getType();
 
-  abstract boolean toBoolNonEmpty();
+  abstract ApproxAccessPathFront toApprox();
 
   TypedContent getHead() { this = TFrontHead(result) }
 }
@@ -1399,7 +1496,7 @@ class AccessPathFrontNil extends AccessPathFront, TFrontNil {
 
   override DataFlowType getType() { result = t }
 
-  override boolean toBoolNonEmpty() { result = false }
+  override ApproxAccessPathFront toApprox() { result = TApproxFrontNil(t) }
 }
 
 class AccessPathFrontHead extends AccessPathFront, TFrontHead {
@@ -1411,7 +1508,7 @@ class AccessPathFrontHead extends AccessPathFront, TFrontHead {
 
   override DataFlowType getType() { result = tc.getContainerType() }
 
-  override boolean toBoolNonEmpty() { result = true }
+  override ApproxAccessPathFront toApprox() { result.getAHead() = tc }
 }
 
 /** An optional access path front. */

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplConsistency.qll

@@ -45,6 +45,16 @@ module Consistency {
     ) {
       none()
     }
+
+    /** Holds if `(c, pos, p)` should be excluded from the consistency test `uniqueParameterNodeAtPosition`. */
+    predicate uniqueParameterNodeAtPositionExclude(DataFlowCallable c, ParameterPosition pos, Node p) {
+      none()
+    }
+
+    /** Holds if `(c, pos, p)` should be excluded from the consistency test `uniqueParameterNodePosition`. */
+    predicate uniqueParameterNodePositionExclude(DataFlowCallable c, ParameterPosition pos, Node p) {
+      none()
+    }
   }
 
   private class RelevantNode extends Node {
@@ -101,9 +111,7 @@ module Consistency {
     exists(int c |
       c =
         strictcount(Node n |
-          not exists(string filepath, int startline, int startcolumn, int endline, int endcolumn |
-            n.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
-          ) and
+          not n.hasLocationInfo(_, _, _, _, _) and
           not any(ConsistencyConfiguration conf).missingLocationExclude(n)
         ) and
       msg = "Nodes without location: " + c
@@ -244,4 +252,27 @@ module Consistency {
     not callable = viableCallable(call) and
     not any(ConsistencyConfiguration c).viableImplInCallContextTooLargeExclude(call, ctx, callable)
   }
+
+  query predicate uniqueParameterNodeAtPosition(
+    DataFlowCallable c, ParameterPosition pos, Node p, string msg
+  ) {
+    not any(ConsistencyConfiguration conf).uniqueParameterNodeAtPositionExclude(c, pos, p) and
+    isParameterNode(p, c, pos) and
+    not exists(unique(Node p0 | isParameterNode(p0, c, pos))) and
+    msg = "Parameters with overlapping positions."
+  }
+
+  query predicate uniqueParameterNodePosition(
+    DataFlowCallable c, ParameterPosition pos, Node p, string msg
+  ) {
+    not any(ConsistencyConfiguration conf).uniqueParameterNodePositionExclude(c, pos, p) and
+    isParameterNode(p, c, pos) and
+    not exists(unique(ParameterPosition pos0 | isParameterNode(p, c, pos0))) and
+    msg = "Parameter node with multiple positions."
+  }
+
+  query predicate uniqueContentApprox(Content c, string msg) {
+    not exists(unique(ContentApprox approx | approx = getContentApprox(c))) and
+    msg = "Non-unique content approximation."
+  }
 }

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll

@@ -819,9 +819,9 @@ predicate attributeReadStep(Node nodeFrom, AttributeContent c, AttrRead nodeTo)
  * synthesized unpacked argument with the name indicated by `c`.
  */
 predicate kwUnpackReadStep(CfgNode nodeFrom, DictionaryElementContent c, Node nodeTo) {
-  exists(CallNode call, CallableValue callable, string name |
+  exists(CallNode call, string name |
     nodeFrom.asCfgNode() = call.getNode().getKwargs().getAFlowNode() and
-    nodeTo = TKwUnpackedNode(call, callable, name) and
+    nodeTo = TKwUnpackedNode(call, _, name) and
     name = c.getKey()
   )
 }
@@ -948,3 +948,10 @@ predicate additionalLambdaFlowStep(Node nodeFrom, Node nodeTo, boolean preserves
 predicate allowParameterReturnInSelf(ParameterNode p) {
   FlowSummaryImpl::Private::summaryAllowParameterReturnInSelf(p)
 }
+
+/** An approximated `Content`. */
+class ContentApprox = Unit;
+
+/** Gets an approximated value for content `c`. */
+pragma[inline]
+ContentApprox getContentApprox(Content c) { any() }

python/ql/lib/semmle/python/dataflow/new/internal/FlowSummaryImpl.qll

@@ -39,6 +39,11 @@ module Public {
       )
       or
       exists(ReturnKind rk | this = TReturnSummaryComponent(rk) and result = "return (" + rk + ")")
+      or
+      exists(SummaryComponent::SyntheticGlobal sg |
+        this = TSyntheticGlobalSummaryComponent(sg) and
+        result = "synthetic global (" + sg + ")"
+      )
     }
   }
 
@@ -159,24 +164,24 @@ module Public {
     SummaryComponentStack return(ReturnKind rk) { result = singleton(SummaryComponent::return(rk)) }
   }
 
-  private predicate noComponentSpecificCsv(SummaryComponent sc) {
-    not exists(getComponentSpecificCsv(sc))
+  private predicate noComponentSpecific(SummaryComponent sc) {
+    not exists(getComponentSpecific(sc))
   }
 
   /** Gets a textual representation of this component used for flow summaries. */
-  private string getComponentCsv(SummaryComponent sc) {
-    result = getComponentSpecificCsv(sc)
+  private string getComponent(SummaryComponent sc) {
+    result = getComponentSpecific(sc)
     or
-    noComponentSpecificCsv(sc) and
+    noComponentSpecific(sc) and
     (
       exists(ArgumentPosition pos |
         sc = TParameterSummaryComponent(pos) and
-        result = "Parameter[" + getArgumentPositionCsv(pos) + "]"
+        result = "Parameter[" + getArgumentPosition(pos) + "]"
       )
       or
       exists(ParameterPosition pos |
         sc = TArgumentSummaryComponent(pos) and
-        result = "Argument[" + getParameterPositionCsv(pos) + "]"
+        result = "Argument[" + getParameterPosition(pos) + "]"
       )
       or
       sc = TReturnSummaryComponent(getReturnValueKind()) and result = "ReturnValue"
@@ -184,16 +189,16 @@ module Public {
   }
 
   /** Gets a textual representation of this stack used for flow summaries. */
-  string getComponentStackCsv(SummaryComponentStack stack) {
+  string getComponentStack(SummaryComponentStack stack) {
     exists(SummaryComponent head, SummaryComponentStack tail |
       head = stack.head() and
       tail = stack.tail() and
-      result = getComponentStackCsv(tail) + "." + getComponentCsv(head)
+      result = getComponentStack(tail) + "." + getComponent(head)
     )
     or
     exists(SummaryComponent c |
       stack = TSingletonSummaryComponentStack(c) and
-      result = getComponentCsv(c)
+      result = getComponent(c)
     )
   }
 
@@ -241,19 +246,39 @@ module Public {
     }
 
     /**
-     * Holds if the summary is auto generated.
+     * Holds if all the summaries that apply to `this` are auto generated and not manually created.
      */
-    predicate isAutoGenerated() { none() }
-  }
-
-  /** A callable with a flow summary stating there is no flow via the callable. */
-  class NegativeSummarizedCallable extends SummarizedCallableBase {
-    NegativeSummarizedCallable() { negativeSummaryElement(this, _) }
+    final predicate isAutoGenerated() { this.hasProvenance("generated") and not this.isManual() }
 
     /**
-     *  Holds if the negative summary is auto generated.
+     * Holds if there exists a manual summary that applies to `this`.
      */
-    predicate isAutoGenerated() { negativeSummaryElement(this, true) }
+    final predicate isManual() { this.hasProvenance("manual") }
+
+    /**
+     * Holds if there exists a summary that applies to `this` that has provenance `provenance`.
+     */
+    predicate hasProvenance(string provenance) { none() }
+  }
+
+  /** A callable where there is no flow via the callable. */
+  class NeutralCallable extends SummarizedCallableBase {
+    NeutralCallable() { neutralElement(this, _) }
+
+    /**
+     * Holds if the neutral is auto generated.
+     */
+    predicate isAutoGenerated() { neutralElement(this, "generated") }
+
+    /**
+     * Holds if there exists a manual neutral that applies to `this`.
+     */
+    final predicate isManual() { this.hasProvenance("manual") }
+
+    /**
+     * Holds if the neutral has provenance `provenance`.
+     */
+    predicate hasProvenance(string provenance) { neutralElement(this, provenance) }
   }
 }
 
@@ -568,10 +593,8 @@ module Private {
           head = TWithContentSummaryComponent(cont)
         )
         or
-        exists(ContentSet cont |
-          head = TWithoutContentSummaryComponent(cont) and
-          result = getNodeType(summaryNodeInputState(c, s.tail()))
-        )
+        head = TWithoutContentSummaryComponent(_) and
+        result = getNodeType(summaryNodeInputState(c, s.tail()))
         or
         exists(ReturnKind rk |
           head = TReturnSummaryComponent(rk) and
@@ -646,8 +669,8 @@ module Private {
 
   /** Holds if summary node `ret` is a return node of kind `rk`. */
   predicate summaryReturnNode(Node ret, ReturnKind rk) {
-    exists(SummarizedCallable callable, SummaryComponentStack s |
-      ret = summaryNodeOutputState(callable, s) and
+    exists(SummaryComponentStack s |
+      ret = summaryNodeOutputState(_, s) and
       s = TSingletonSummaryComponentStack(TReturnSummaryComponent(rk))
     )
   }
@@ -987,12 +1010,12 @@ module Private {
       private predicate relevantSummaryElementGenerated(
         AccessPath inSpec, AccessPath outSpec, string kind
       ) {
-        summaryElement(this, inSpec, outSpec, kind, true) and
-        not summaryElement(this, _, _, _, false)
+        summaryElement(this, inSpec, outSpec, kind, "generated") and
+        not summaryElement(this, _, _, _, "manual")
       }
 
       private predicate relevantSummaryElement(AccessPath inSpec, AccessPath outSpec, string kind) {
-        summaryElement(this, inSpec, outSpec, kind, false)
+        summaryElement(this, inSpec, outSpec, kind, "manual")
         or
         this.relevantSummaryElementGenerated(inSpec, outSpec, kind)
       }
@@ -1011,7 +1034,9 @@ module Private {
         )
       }
 
-      override predicate isAutoGenerated() { this.relevantSummaryElementGenerated(_, _, _) }
+      override predicate hasProvenance(string provenance) {
+        summaryElement(this, _, _, _, provenance)
+      }
     }
 
     /** Holds if component `c` of specification `spec` cannot be parsed. */
@@ -1161,9 +1186,9 @@ module Private {
       string toString() { result = super.toString() }
     }
 
-    /** A flow summary to include in the `negativeSummary/1` query predicate. */
-    abstract class RelevantNegativeSummarizedCallable instanceof NegativeSummarizedCallable {
-      /** Gets the string representation of this callable used by `summary/1`. */
+    /** A model to include in the `neutral/1` query predicate. */
+    abstract class RelevantNeutralCallable instanceof NeutralCallable {
+      /** Gets the string representation of this callable used by `neutral/1`. */
       abstract string getCallableCsv();
 
       string toString() { result = super.toString() }
@@ -1180,13 +1205,13 @@ module Private {
       if c.isAutoGenerated() then result = "generated" else result = "manual"
     }
 
-    private string renderProvenanceNegative(NegativeSummarizedCallable c) {
+    private string renderProvenanceNeutral(NeutralCallable c) {
       if c.isAutoGenerated() then result = "generated" else result = "manual"
     }
 
     /**
      * A query predicate for outputting flow summaries in semi-colon separated format in QL tests.
-     * The syntax is: "namespace;type;overrides;name;signature;ext;inputspec;outputspec;kind;provenance"",
+     * The syntax is: "namespace;type;overrides;name;signature;ext;inputspec;outputspec;kind;provenance",
      * ext is hardcoded to empty.
      */
     query predicate summary(string csv) {
@@ -1197,22 +1222,22 @@ module Private {
         c.relevantSummary(input, output, preservesValue) and
         csv =
           c.getCallableCsv() // Callable information
-            + getComponentStackCsv(input) + ";" // input
-            + getComponentStackCsv(output) + ";" // output
+            + getComponentStack(input) + ";" // input
+            + getComponentStack(output) + ";" // output
             + renderKind(preservesValue) + ";" // kind
             + renderProvenance(c) // provenance
       )
     }
 
     /**
-     * Holds if a negative flow summary `csv` exists (semi-colon separated format). Used for testing purposes.
+     * Holds if a neutral model `csv` exists (semi-colon separated format). Used for testing purposes.
      * The syntax is: "namespace;type;name;signature;provenance"",
      */
-    query predicate negativeSummary(string csv) {
-      exists(RelevantNegativeSummarizedCallable c |
+    query predicate neutral(string csv) {
+      exists(RelevantNeutralCallable c |
         csv =
           c.getCallableCsv() // Callable information
-            + renderProvenanceNegative(c) // provenance
+            + renderProvenanceNeutral(c) // provenance
       )
     }
   }

python/ql/lib/semmle/python/dataflow/new/internal/FlowSummaryImplSpecific.qll

@@ -78,25 +78,24 @@ DataFlowType getSyntheticGlobalType(SummaryComponent::SyntheticGlobal sg) { any(
 
 /**
  * Holds if an external flow summary exists for `c` with input specification
- * `input`, output specification `output`, kind `kind`, and a flag `generated`
- * stating whether the summary is autogenerated.
+ * `input`, output specification `output`, kind `kind`, and provenance `provenance`.
  */
 predicate summaryElement(
-  FlowSummary::SummarizedCallable c, string input, string output, string kind, boolean generated
+  FlowSummary::SummarizedCallable c, string input, string output, string kind, string provenance
 ) {
   exists(boolean preservesValue |
     c.propagatesFlowExt(input, output, preservesValue) and
     (if preservesValue = true then kind = "value" else kind = "taint") and
-    generated = false
+    provenance = "manual"
   )
 }
 
 /**
- * Holds if a negative flow summary exists for `c`, which means that there is no
- * flow through `c`. The flag `generated` states whether the summary is autogenerated.
- * Note. Negative flow summaries has not been implemented for Python.
+ * Holds if a neutral model exists for `c` with provenance `provenance`,
+ * which means that there is no flow through `c`.
+ * Note. Neutral models have not been implemented for Python.
  */
-predicate negativeSummaryElement(FlowSummary::SummarizedCallable c, boolean generated) { none() }
+predicate neutralElement(FlowSummary::SummarizedCallable c, string provenance) { none() }
 
 /**
  * Gets the summary component for specification component `c`, if any.
@@ -109,16 +108,16 @@ SummaryComponent interpretComponentSpecific(AccessPathToken c) {
 }
 
 /** Gets the textual representation of a summary component in the format used for flow summaries. */
-string getComponentSpecificCsv(SummaryComponent sc) {
+string getComponentSpecific(SummaryComponent sc) {
   sc = TContentSummaryComponent(any(ListElementContent c)) and
   result = "ListElement"
 }
 
 /** Gets the textual representation of a parameter position in the format used for flow summaries. */
-string getParameterPositionCsv(ParameterPosition pos) { result = pos.toString() }
+string getParameterPosition(ParameterPosition pos) { result = pos.toString() }
 
 /** Gets the textual representation of an argument position in the format used for flow summaries. */
-string getArgumentPositionCsv(ArgumentPosition pos) { result = pos.toString() }
+string getArgumentPosition(ArgumentPosition pos) { result = pos.toString() }
 
 /** Holds if input specification component `c` needs a reference. */
 predicate inputNeedsReferenceSpecific(string c) { none() }
@@ -137,17 +136,15 @@ ReturnKind getReturnValueKind() { any() }
 private module UnusedSourceSinkInterpretation {
   /**
    * Holds if an external source specification exists for `n` with output specification
-   * `output`, kind `kind`, and a flag `generated` stating whether the source specification is
-   * autogenerated.
+   * `output`, kind `kind`, and provenance `provenance`.
    */
-  predicate sourceElement(AstNode n, string output, string kind, boolean generated) { none() }
+  predicate sourceElement(AstNode n, string output, string kind, string provenance) { none() }
 
   /**
    * Holds if an external sink specification exists for `n` with input specification
-   * `input`, kind `kind` and a flag `generated` stating whether the sink specification is
-   * autogenerated.
+   * `input`, kind `kind` and provenance `provenance`.
    */
-  predicate sinkElement(AstNode n, string input, string kind, boolean generated) { none() }
+  predicate sinkElement(AstNode n, string input, string kind, string provenance) { none() }
 
   class SourceOrSinkElement = AstNode;
 

python/ql/lib/semmle/python/dataflow/new/internal/IterableUnpacking.qll

@@ -207,16 +207,13 @@ class AssignmentTarget extends ControlFlowNode {
 }
 
 /** A direct (or top-level) target of an unpacking assignment. */
-class UnpackingAssignmentDirectTarget extends ControlFlowNode {
+class UnpackingAssignmentDirectTarget extends ControlFlowNode instanceof SequenceNode {
   Expr value;
 
   UnpackingAssignmentDirectTarget() {
-    this instanceof SequenceNode and
-    (
-      value = this.(AssignmentTarget).getValue()
-      or
-      value = this.(ForTarget).getSource()
-    )
+    value = this.(AssignmentTarget).getValue()
+    or
+    value = this.(ForTarget).getSource()
   }
 
   Expr getValue() { result = value }

python/ql/lib/semmle/python/dataflow/old/Implementation.qll

@@ -197,9 +197,7 @@ class TaintTrackingNode extends TTaintTrackingNode {
  * It is implemented as a separate class for clarity and to keep the code
  * in `TaintTracking::Configuration` simpler.
  */
-class TaintTrackingImplementation extends string {
-  TaintTrackingImplementation() { this instanceof TaintTracking::Configuration }
-
+class TaintTrackingImplementation extends string instanceof TaintTracking::Configuration {
   /**
    * Hold if there is a flow from `source`, which is a taint source, to
    * `sink`, which is a taint sink, with this configuration.
@@ -218,7 +216,7 @@ class TaintTrackingImplementation extends string {
   ) {
     context = TNoParam() and
     path = TNoAttribute() and
-    this.(TaintTracking::Configuration).isSource(node, kind)
+    super.isSource(node, kind)
   }
 
   /** Hold if `source` is a source of taint. */
@@ -234,7 +232,7 @@ class TaintTrackingImplementation extends string {
     exists(DataFlow::Node node, AttributePath path, TaintKind kind |
       sink = TTaintTrackingNode_(node, _, path, kind, this) and
       path = TNoAttribute() and
-      this.(TaintTracking::Configuration).isSink(node, kind)
+      super.isSink(node, kind)
     )
   }
 
@@ -259,11 +257,11 @@ class TaintTrackingImplementation extends string {
   ) {
     this.unprunedStep(src, node, context, path, kind, edgeLabel) and
     node.getBasicBlock().likelyReachable() and
-    not this.(TaintTracking::Configuration).isBarrier(node) and
+    not super.isBarrier(node) and
     (
       not path = TNoAttribute()
       or
-      not this.(TaintTracking::Configuration).isBarrier(node, kind) and
+      not super.isBarrier(node, kind) and
       exists(DataFlow::Node srcnode, TaintKind srckind |
         src = TTaintTrackingNode_(srcnode, _, _, srckind, this) and
         not this.prunedEdge(srcnode, node, srckind, kind)
@@ -274,9 +272,9 @@ class TaintTrackingImplementation extends string {
   private predicate prunedEdge(
     DataFlow::Node srcnode, DataFlow::Node destnode, TaintKind srckind, TaintKind destkind
   ) {
-    this.(TaintTracking::Configuration).isBarrierEdge(srcnode, destnode, srckind, destkind)
+    super.isBarrierEdge(srcnode, destnode, srckind, destkind)
     or
-    srckind = destkind and this.(TaintTracking::Configuration).isBarrierEdge(srcnode, destnode)
+    srckind = destkind and super.isBarrierEdge(srcnode, destnode)
   }
 
   private predicate unprunedStep(
@@ -314,14 +312,14 @@ class TaintTrackingImplementation extends string {
     this.legacyExtensionStep(src, node, context, path, kind, edgeLabel)
     or
     exists(DataFlow::Node srcnode, TaintKind srckind |
-      this.(TaintTracking::Configuration).isAdditionalFlowStep(srcnode, node, srckind, kind) and
+      super.isAdditionalFlowStep(srcnode, node, srckind, kind) and
       src = TTaintTrackingNode_(srcnode, context, path, srckind, this) and
       path.noAttribute() and
       edgeLabel = "additional with kind"
     )
     or
     exists(DataFlow::Node srcnode |
-      this.(TaintTracking::Configuration).isAdditionalFlowStep(srcnode, node) and
+      super.isAdditionalFlowStep(srcnode, node) and
       src = TTaintTrackingNode_(srcnode, context, path, kind, this) and
       path.noAttribute() and
       edgeLabel = "additional"
@@ -533,8 +531,8 @@ class TaintTrackingImplementation extends string {
     TaintTrackingNode src, DataFlow::Node node, TaintTrackingContext context, AttributePath path,
     TaintKind kind, string edgeLabel
   ) {
-    exists(DataFlow::Node srcnode, CallNode call, TaintKind srckind, string name |
-      src = TTaintTrackingNode_(srcnode, context, path, srckind, this) and
+    exists(CallNode call, TaintKind srckind, string name |
+      src = TTaintTrackingNode_(_, context, path, srckind, this) and
       call.getFunction().(AttrNode).getObject(name) = src.getNode().asCfgNode() and
       kind = srckind.getTaintOfMethodResult(name) and
       node.asCfgNode() = call
@@ -562,8 +560,8 @@ class TaintTrackingImplementation extends string {
     TaintTrackingNode src, DataFlow::Node node, TaintTrackingContext context, AttributePath path,
     TaintKind kind, string edgeLabel
   ) {
-    exists(CallNode call, PythonFunctionObjectInternal pyfunc, int arg |
-      this.callWithTaintedArgument(src, call, _, pyfunc, arg, path, kind) and
+    exists(PythonFunctionObjectInternal pyfunc, int arg |
+      this.callWithTaintedArgument(src, _, _, pyfunc, arg, path, kind) and
       node.asCfgNode() = pyfunc.getParameter(arg) and
       context = TParamContext(kind, path, arg)
     ) and
@@ -618,7 +616,7 @@ class TaintTrackingImplementation extends string {
     TaintKind kind, string edgeLabel
   ) {
     exists(TaintTracking::Extension extension, DataFlow::Node srcnode, TaintKind srckind |
-      this.(TaintTracking::Configuration).isExtension(extension) and
+      super.isExtension(extension) and
       src = TTaintTrackingNode_(srcnode, context, path, srckind, this) and
       srcnode.asCfgNode() = extension
     |
@@ -646,9 +644,7 @@ class TaintTrackingImplementation extends string {
  * Another taint-tracking class to help partition the code for clarity
  * This class handle tracking of ESSA variables.
  */
-private class EssaTaintTracking extends string {
-  EssaTaintTracking() { this instanceof TaintTracking::Configuration }
-
+private class EssaTaintTracking extends string instanceof TaintTracking::Configuration {
   pragma[noinline]
   predicate taintedDefinition(
     TaintTrackingNode src, EssaDefinition defn, TaintTrackingContext context, AttributePath path,
@@ -691,7 +687,7 @@ private class EssaTaintTracking extends string {
       defn = phi.asVariable().getDefinition() and
       predvar = defn.getInput(pred) and
       not pred.unlikelySuccessor(defn.getBasicBlock()) and
-      not this.(TaintTracking::Configuration).isBarrierEdge(srcnode, phi) and
+      not super.isBarrierEdge(srcnode, phi) and
       srcnode.asVariable() = predvar
     )
   }
@@ -781,7 +777,7 @@ private class EssaTaintTracking extends string {
     exists(DataFlow::Node srcnode |
       src = TTaintTrackingNode_(srcnode, context, path, kind, this) and
       srcnode.asVariable() = defn.getInput() and
-      not this.(TaintTracking::Configuration).isBarrierTest(defn.getTest(), defn.getSense())
+      not super.isBarrierTest(defn.getTest(), defn.getSense())
     )
   }
 
@@ -799,11 +795,9 @@ private class EssaTaintTracking extends string {
     TaintTrackingNode src, PyEdgeRefinement defn, TaintTrackingContext context, AttributePath path,
     TaintKind kind
   ) {
-    exists(DataFlow::Node srcnode, ControlFlowNode use |
-      src = TTaintTrackingNode_(srcnode, context, path, kind, this) and
-      not this.(TaintTracking::Configuration).isBarrierTest(defn.getTest(), defn.getSense()) and
-      defn.getSense() = this.testEvaluates(defn, defn.getTest(), use, src)
-    )
+    src = TTaintTrackingNode_(_, context, path, kind, this) and
+    not super.isBarrierTest(defn.getTest(), defn.getSense()) and
+    defn.getSense() = this.testEvaluates(defn, defn.getTest(), _, src)
   }
 
   pragma[noinline]
@@ -815,7 +809,7 @@ private class EssaTaintTracking extends string {
       src = TTaintTrackingNode_(srcnode, context, path, kind, this) and
       piNodeTestAndUse(defn, test, use) and
       srcnode.asVariable() = defn.getInput() and
-      not this.(TaintTracking::Configuration).isBarrierTest(test, defn.getSense()) and
+      not super.isBarrierTest(test, defn.getSense()) and
       testEvaluatesMaybe(test, use)
     )
   }

python/ql/lib/semmle/python/dataflow/old/StateTracking.qll

@@ -99,9 +99,7 @@ module StateTracking {
         exists(int n |
           f = b.getNode(n) and
           appliesToNode(state, b.getNode(n - 1), ctx, sense) and
-          not exists(PythonFunctionObjectInternal func, Context callee |
-            callee.fromCall(f, func, ctx)
-          )
+          not exists(Context callee | callee.fromCall(f, _, ctx))
         )
       )
       or

python/ql/lib/semmle/python/essa/Definitions.qll

@@ -69,6 +69,8 @@ abstract class SsaSourceVariable extends @py_variable {
     or
     SsaSource::exception_capture(this, def)
     or
+    SsaSource::exception_group_capture(this, def)
+    or
     SsaSource::with_definition(this, def)
     or
     SsaSource::pattern_capture_definition(this, def)
@@ -205,22 +207,19 @@ class BuiltinVariable extends SsaSourceVariable {
   override CallNode redefinedAtCallSite() { none() }
 }
 
-class ModuleVariable extends SsaSourceVariable {
+class ModuleVariable extends SsaSourceVariable instanceof GlobalVariable {
   ModuleVariable() {
-    this instanceof GlobalVariable and
-    (
-      exists(this.(Variable).getAStore())
-      or
-      this.(Variable).getId() = "__name__"
-      or
-      this.(Variable).getId() = "__package__"
-      or
-      exists(ImportStar is | is.getScope() = this.(Variable).getScope())
-    )
+    exists(this.(Variable).getAStore())
+    or
+    this.(Variable).getId() = "__name__"
+    or
+    this.(Variable).getId() = "__package__"
+    or
+    exists(ImportStar is | is.getScope() = this.(Variable).getScope())
   }
 
   pragma[nomagic]
-  private Scope scope_as_global_variable() { result = this.(GlobalVariable).getScope() }
+  private Scope scope_as_global_variable() { result = GlobalVariable.super.getScope() }
 
   pragma[noinline]
   CallNode global_variable_callnode() { result.getScope() = this.scope_as_global_variable() }
@@ -263,7 +262,7 @@ class ModuleVariable extends SsaSourceVariable {
       class_with_global_metaclass(s, this)
       or
       /* Variable is used in scope */
-      this.(GlobalVariable).getAUse().getScope() = s
+      GlobalVariable.super.getAUse().getScope() = s
     )
     or
     exists(ImportTimeScope scope | scope.entryEdge(_, result) |

python/ql/lib/semmle/python/essa/Essa.qll

@@ -108,7 +108,7 @@ private string location_string(EssaVariable v) {
 
 /* Helper to compute an index for this SSA variable. */
 private int var_index(EssaVariable v) {
-  location_string(v) = rank[result](string s | exists(EssaVariable x | location_string(x) = s) | s)
+  location_string(v) = rank[result](string s | location_string(_) = s | s)
 }
 
 /* Helper for `v.getRepresentation()` */
@@ -511,12 +511,16 @@ class AssignmentDefinition extends EssaNodeDefinition {
   override string getAPrimaryQlClass() { result = "AssignmentDefinition" }
 }
 
-/** A capture of a raised exception `except ExceptionType ex:` */
+/** A capture of a raised exception `except ExceptionType as ex:` */
 class ExceptionCapture extends EssaNodeDefinition {
   ExceptionCapture() {
     SsaSource::exception_capture(this.getSourceVariable(), this.getDefiningNode())
   }
 
+  /**
+   * Gets the type handled by this exception handler
+   * `ExceptionType` in `except ExceptionType as ex:`.
+   */
   ControlFlowNode getType() {
     exists(ExceptFlowNode ex |
       ex.getName() = this.getDefiningNode() and
@@ -529,6 +533,28 @@ class ExceptionCapture extends EssaNodeDefinition {
   override string getAPrimaryQlClass() { result = "ExceptionCapture" }
 }
 
+/** A capture of a raised exception group `except* ExceptionType as ex:` */
+class ExceptionGroupCapture extends EssaNodeDefinition {
+  ExceptionGroupCapture() {
+    SsaSource::exception_group_capture(this.getSourceVariable(), this.getDefiningNode())
+  }
+
+  /**
+   * Gets the type handled by this exception handler
+   * `ExceptionType` in `except* ExceptionType as ex:`.
+   */
+  ControlFlowNode getType() {
+    exists(ExceptGroupFlowNode ex |
+      ex.getName() = this.getDefiningNode() and
+      result = ex.getType()
+    )
+  }
+
+  override string getRepresentation() { result = "except* " + this.getSourceVariable().getName() }
+
+  override string getAPrimaryQlClass() { result = "ExceptionGroupCapture" }
+}
+
 /** An assignment to a variable as part of a multiple assignment `..., v, ... = val` */
 class MultiAssignmentDefinition extends EssaNodeDefinition {
   MultiAssignmentDefinition() {

python/ql/lib/semmle/python/essa/SsaDefinitions.qll

@@ -30,6 +30,13 @@ module SsaSource {
     exists(ExceptFlowNode ex | ex.getName() = defn)
   }
 
+  /** Holds if `v` is defined by assignment of the captured exception group. */
+  cached
+  predicate exception_group_capture(Variable v, NameNode defn) {
+    defn.defines(v) and
+    exists(ExceptGroupFlowNode ex | ex.getName() = defn)
+  }
+
   /** Holds if `v` is defined by a with statement. */
   cached
   predicate with_definition(Variable v, ControlFlowNode defn) {

python/ql/lib/semmle/python/frameworks/Aiohttp.qll

@@ -59,24 +59,24 @@ module AiohttpWebModel {
    * Extend this class to refine existing API models. If you want to model new APIs,
    * extend `AiohttpRouteSetup::Range` instead.
    */
-  class AiohttpRouteSetup extends Http::Server::RouteSetup::Range {
-    AiohttpRouteSetup::Range range;
-
-    AiohttpRouteSetup() { this = range }
-
+  class AiohttpRouteSetup extends Http::Server::RouteSetup::Range instanceof AiohttpRouteSetup::Range {
     override Parameter getARoutedParameter() { none() }
 
     override string getFramework() { result = "aiohttp.web" }
 
     /** Gets the argument specifying the handler (either a coroutine or a view-class). */
-    DataFlow::Node getHandlerArg() { result = range.getHandlerArg() }
+    DataFlow::Node getHandlerArg() { result = super.getHandlerArg() }
 
-    override DataFlow::Node getUrlPatternArg() { result = range.getUrlPatternArg() }
+    override DataFlow::Node getUrlPatternArg() {
+      result = AiohttpRouteSetup::Range.super.getUrlPatternArg()
+    }
 
     /** Gets the view-class that is referenced in the view-class handler argument, if any. */
-    Class getViewClass() { result = range.getViewClass() }
+    Class getViewClass() { result = super.getViewClass() }
 
-    override Function getARequestHandler() { result = range.getARequestHandler() }
+    override Function getARequestHandler() {
+      result = AiohttpRouteSetup::Range.super.getARequestHandler()
+    }
   }
 
   /** Provides a class for modeling new aiohttp.web route setups. */

python/ql/lib/semmle/python/frameworks/Django.qll

@@ -2526,11 +2526,10 @@ module PrivateDjango {
    *
    * Needs this subclass to be considered a RegexString.
    */
-  private class DjangoRouteRegex extends RegexString {
+  private class DjangoRouteRegex extends RegexString instanceof StrConst {
     DjangoRegexRouteSetup rePathCall;
 
     DjangoRouteRegex() {
-      this instanceof StrConst and
       rePathCall.getUrlPatternArg().getALocalSource() = DataFlow::exprNode(this)
     }
 

python/ql/lib/semmle/python/frameworks/Stdlib.qll

@@ -1163,13 +1163,16 @@ private module StdlibPrivate {
   API::Node subprocess() { result = API::moduleImport("subprocess") }
 
   /**
-   * A call to `subprocess.Popen` or helper functions (call, check_call, check_output, run)
+   * A call to `subprocess.Popen` or helper functions (call, check_call, check_output, run, getoutput, getstatusoutput)
    * See https://docs.python.org/3.8/library/subprocess.html#subprocess.Popen
+   * ref: https://docs.python.org/3/library/subprocess.html#legacy-shell-invocation-functions
    */
   private class SubprocessPopenCall extends SystemCommandExecution::Range, DataFlow::CallCfgNode {
     SubprocessPopenCall() {
       exists(string name |
-        name in ["Popen", "call", "check_call", "check_output", "run"] and
+        name in [
+            "Popen", "call", "check_call", "check_output", "run", "getoutput", "getstatusoutput"
+          ] and
         this = subprocess().getMember(name).getACall()
       )
     }

python/ql/lib/semmle/python/frameworks/Tornado.qll

@@ -385,13 +385,10 @@ module Tornado {
    *
    * Needs this subclass to be considered a RegexString.
    */
-  private class TornadoRouteRegex extends RegexString {
+  private class TornadoRouteRegex extends RegexString instanceof StrConst {
     TornadoRouteSetup setup;
 
-    TornadoRouteRegex() {
-      this instanceof StrConst and
-      setup.getUrlPatternArg().getALocalSource() = DataFlow::exprNode(this)
-    }
+    TornadoRouteRegex() { setup.getUrlPatternArg().getALocalSource() = DataFlow::exprNode(this) }
 
     TornadoRouteSetup getRouteSetup() { result = setup }
   }

python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModels.qll

@@ -72,6 +72,7 @@ private module API = Specific::API;
 private module DataFlow = Specific::DataFlow;
 
 private import Specific::AccessPathSyntax
+private import ApiGraphModelsExtensions as Extensions
 
 /** Module containing hooks for providing input data to be interpreted as a model. */
 module ModelInput {
@@ -236,6 +237,8 @@ predicate sourceModel(string type, string path, string kind) {
     row.splitAt(";", 1) = path and
     row.splitAt(";", 2) = kind
   )
+  or
+  Extensions::sourceModel(type, path, kind)
 }
 
 /** Holds if a sink model exists for the given parameters. */
@@ -246,6 +249,8 @@ private predicate sinkModel(string type, string path, string kind) {
     row.splitAt(";", 1) = path and
     row.splitAt(";", 2) = kind
   )
+  or
+  Extensions::sinkModel(type, path, kind)
 }
 
 /** Holds if a summary model `row` exists for the given parameters. */
@@ -258,6 +263,8 @@ private predicate summaryModel(string type, string path, string input, string ou
     row.splitAt(";", 3) = output and
     row.splitAt(";", 4) = kind
   )
+  or
+  Extensions::summaryModel(type, path, input, output, kind)
 }
 
 /** Holds if a type model exists for the given parameters. */
@@ -268,6 +275,8 @@ private predicate typeModel(string type1, string type2, string path) {
     row.splitAt(";", 1) = type2 and
     row.splitAt(";", 2) = path
   )
+  or
+  Extensions::typeModel(type1, type2, path)
 }
 
 /** Holds if a type variable model exists for the given parameters. */
@@ -277,6 +286,8 @@ private predicate typeVariableModel(string name, string path) {
     row.splitAt(";", 0) = name and
     row.splitAt(";", 1) = path
   )
+  or
+  Extensions::typeVariableModel(name, path)
 }
 
 /**

python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModelsExtensions.qll

@@ -0,0 +1,36 @@
+/**
+ * Defines extensible predicates for contributing library models from data extensions.
+ */
+
+/**
+ * Holds if the value at `(type, path)` should be seen as a flow
+ * source of the given `kind`.
+ *
+ * The kind `remote` represents a general remote flow source.
+ */
+extensible predicate sourceModel(string type, string path, string kind);
+
+/**
+ * Holds if the value at `(type, path)` should be seen as a sink
+ * of the given `kind`.
+ */
+extensible predicate sinkModel(string type, string path, string kind);
+
+/**
+ * Holds if calls to `(type, path)`, the value referred to by `input`
+ * can flow to the value referred to by `output`.
+ *
+ * `kind` should be either `value` or `taint`, for value-preserving or taint-preserving steps,
+ * respectively.
+ */
+extensible predicate summaryModel(string type, string path, string input, string output, string kind);
+
+/**
+ * Holds if `(type2, path)` should be seen as an instance of `type1`.
+ */
+extensible predicate typeModel(string type1, string type2, string path);
+
+/**
+ * Holds if `path` can be substituted for a token `TypeVar[name]`.
+ */
+extensible predicate typeVariableModel(string name, string path);

python/ql/lib/semmle/python/frameworks/data/internal/model.yml

@@ -0,0 +1,26 @@
+extensions:
+  # Contribute empty data sets to avoid errors about an undefined extensionals
+  - addsTo:
+      pack: codeql/python-all
+      extensible: sourceModel
+    data: []
+
+  - addsTo:
+      pack: codeql/python-all
+      extensible: sinkModel
+    data: []
+
+  - addsTo:
+      pack: codeql/python-all
+      extensible: summaryModel
+    data: []
+
+  - addsTo:
+      pack: codeql/python-all
+      extensible: typeModel
+    data: []
+
+  - addsTo:
+      pack: codeql/python-all
+      extensible: typeVariableModel
+    data: []

python/ql/lib/semmle/python/objects/Constants.qll

@@ -35,10 +35,8 @@ abstract class ConstantObjectInternal extends ObjectInternal {
   pragma[noinline]
   override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) {
     PointsToInternal::attributeRequired(pragma[only_bind_into](this), pragma[only_bind_into](name)) and
-    exists(ObjectInternal cls_attr, CfgOrigin attr_orig |
-      this.getClass()
-          .(ClassObjectInternal)
-          .lookup(pragma[only_bind_into](name), cls_attr, attr_orig) and
+    exists(ObjectInternal cls_attr |
+      this.getClass().(ClassObjectInternal).lookup(pragma[only_bind_into](name), cls_attr, _) and
       cls_attr.isDescriptor() = true and
       cls_attr.descriptorGetInstance(this, value, origin)
     )

python/ql/lib/semmle/python/objects/Instances.qll

@@ -246,8 +246,8 @@ class SelfInstanceInternal extends TSelfInstance, InstanceObject {
 
   pragma[noinline]
   override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
-    exists(AttrNode attr, ClassObjectInternal cls |
-      receiver_type(attr, name, this, cls) and
+    exists(ClassObjectInternal cls |
+      receiver_type(_, name, this, cls) and
       cls_descriptor(cls, name, descriptor)
     ) and
     instance = this
@@ -349,8 +349,8 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal {
 
   pragma[noinline]
   override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) {
-    exists(AttrNode attr, ClassObjectInternal cls |
-      receiver_type(attr, name, this, cls) and
+    exists(ClassObjectInternal cls |
+      receiver_type(_, name, this, cls) and
       cls_descriptor(cls, name, descriptor)
     ) and
     instance = this

python/ql/lib/semmle/python/objects/ObjectAPI.qll

@@ -674,7 +674,7 @@ abstract class FunctionValue extends CallableValue {
   /** Gets the maximum number of parameters that can be correctly passed to this function */
   abstract int maxParameters();
 
-  predicate isOverridingMethod() { exists(Value f | this.overrides(f)) }
+  predicate isOverridingMethod() { this.overrides(_) }
 
   predicate isOverriddenMethod() { exists(Value f | f.overrides(this)) }
 

python/ql/lib/semmle/python/pointsto/PointsTo.qll

@@ -1026,8 +1026,8 @@ module InterProceduralPointsTo {
     ParameterDefinition def, PointsToContext context, ObjectInternal value, ControlFlowNode origin
   ) {
     def.isSelf() and
-    exists(CallNode call, BoundMethodObjectInternal method, Function func, PointsToContext caller |
-      callWithContext(call, caller, method, context) and
+    exists(BoundMethodObjectInternal method, Function func |
+      callWithContext(_, _, method, context) and
       func = method.getScope() and
       def.getScope() = func and
       value = method.getSelf() and
@@ -1062,8 +1062,8 @@ module InterProceduralPointsTo {
   private predicate context_for_default_value(ParameterDefinition def, PointsToContext context) {
     context.isRuntime() and exists(def)
     or
-    exists(PointsToContext caller, CallNode call, PythonFunctionObjectInternal func, int n |
-      context.fromCall(call, func, caller) and
+    exists(CallNode call, PythonFunctionObjectInternal func, int n |
+      context.fromCall(call, func, _) and
       func.getScope().getArg(n) = def.getParameter() and
       not exists(call.getArg(n)) and
       not exists(call.getArgByName(def.getVariable().getName())) and
@@ -1184,9 +1184,7 @@ module InterProceduralPointsTo {
     PointsToContext callee
   ) {
     PointsToInternal::pointsTo(argument, caller, _, _) and
-    exists(CallNode call, Function func, int offset |
-      callsite_calls_function(call, caller, func, callee, offset)
-    |
+    exists(CallNode call, Function func | callsite_calls_function(call, caller, func, callee, _) |
       exists(string name |
         argument = call.getArgByName(name) and
         function_parameter_name(func, param, name)
@@ -1460,10 +1458,9 @@ module Expressions {
     SubscriptNode subscr, PointsToContext context, ObjectInternal value, ControlFlowNode obj,
     ObjectInternal objvalue
   ) {
-    exists(ControlFlowNode index | subscriptObjectAndIndex(subscr, context, obj, objvalue, index) |
-      objvalue.subscriptUnknown() and
-      value = ObjectInternal::unknown()
-    )
+    subscriptObjectAndIndex(subscr, context, obj, objvalue, _) and
+    objvalue.subscriptUnknown() and
+    value = ObjectInternal::unknown()
     or
     exists(int n |
       subscriptObjectAndIndexPointsToInt(subscr, context, obj, objvalue, n) and
@@ -1567,9 +1564,9 @@ module Expressions {
       b.operands(other, op, operand)
     |
       op instanceof BitOr and
-      exists(ObjectInternal obj, int i1, int i2 |
+      exists(int i1, int i2 |
         pointsToInt(operand, context, opvalue, i1) and
-        pointsToInt(other, context, obj, i2) and
+        pointsToInt(other, context, _, i2) and
         value = TInt(i1.bitOr(i2))
       )
     )
@@ -2071,7 +2068,7 @@ module Expressions {
     exists(ObjectInternal sup_or_tuple |
       issubclass_call(_, _, _, sub, sup_or_tuple) and sub.isClass() = true
       or
-      exists(ObjectInternal val | isinstance_call(_, _, _, val, sub, sup_or_tuple))
+      isinstance_call(_, _, _, _, sub, sup_or_tuple)
     |
       sup = sup_or_tuple
       or
@@ -2759,8 +2756,8 @@ module ModuleAttributes {
     )
     or
     /* Retain value held before import */
-    exists(ModuleObjectInternal mod, EssaVariable input |
-      importStarDef(def, input, mod) and
+    exists(ModuleObjectInternal mod |
+      importStarDef(def, _, mod) and
       (InterModulePointsTo::moduleExportsBoolean(mod, name) = false or name.charAt(0) = "_") and
       attributePointsTo(def.getInput(), name, value, origin)
     )
@@ -2787,8 +2784,8 @@ module ModuleAttributes {
     CallsiteRefinement def, string name, ObjectInternal value, CfgOrigin origin
   ) {
     def.getVariable().isMetaVariable() and
-    exists(EssaVariable var, Function func, PointsToContext callee |
-      InterProceduralPointsTo::callsite_calls_function(def.getCall(), _, func, callee, _) and
+    exists(EssaVariable var, Function func |
+      InterProceduralPointsTo::callsite_calls_function(def.getCall(), _, func, _, _) and
       var = moduleStateVariable(func.getANormalExit()) and
       attributePointsTo(var, name, value, origin)
     )

python/ql/lib/semmle/python/regex.qll

@@ -994,10 +994,8 @@ abstract class RegexString extends Expr {
       this.specialCharacter(end, y, ["$", "\\Z"])
     )
     or
-    exists(int x |
-      this.lastPart(x, end) and
-      this.item(start, end)
-    )
+    this.lastPart(_, end) and
+    this.item(start, end)
     or
     exists(int y | this.lastPart(start, y) | this.qualifiedPart(start, end, y, _, _))
     or

python/ql/lib/semmle/python/security/dataflow/PamAuthorizationCustomizations.qll

@@ -0,0 +1,61 @@
+/**
+ * Provides default sources, sinks and sanitizers for detecting
+ * "PAM Authorization" vulnerabilities.
+ */
+
+import python
+import semmle.python.ApiGraphs
+import semmle.python.dataflow.new.TaintTracking
+import semmle.python.dataflow.new.RemoteFlowSources
+
+/**
+ * Provides default sources, sinks and sanitizers for detecting
+ * "PAM Authorization" vulnerabilities.
+ */
+module PamAuthorizationCustomizations {
+  /**
+   * Models a node corresponding to the `pam` library
+   */
+  API::Node libPam() {
+    exists(API::CallNode findLibCall, API::CallNode cdllCall |
+      findLibCall =
+        API::moduleImport("ctypes").getMember("util").getMember("find_library").getACall() and
+      findLibCall.getParameter(0).getAValueReachingSink().asExpr().(StrConst).getText() = "pam" and
+      cdllCall = API::moduleImport("ctypes").getMember("CDLL").getACall() and
+      cdllCall.getParameter(0).getAValueReachingSink() = findLibCall
+    |
+      result = cdllCall.getReturn()
+    )
+  }
+
+  /**
+   * A data flow source for "PAM Authorization" vulnerabilities.
+   */
+  abstract class Source extends DataFlow::Node { }
+
+  /**
+   * A data flow sink for "PAM Authorization" vulnerabilities.
+   */
+  abstract class Sink extends DataFlow::Node { }
+
+  /**
+   * A source of remote user input, considered as a flow source.
+   */
+  class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { }
+
+  /**
+   * A vulnerable `pam_authenticate` call considered as a flow sink.
+   */
+  class VulnPamAuthCall extends API::CallNode, Sink {
+    VulnPamAuthCall() {
+      exists(DataFlow::Node h |
+        this = libPam().getMember("pam_authenticate").getACall() and
+        h = this.getArg(0) and
+        not exists(API::CallNode acctMgmtCall |
+          acctMgmtCall = libPam().getMember("pam_acct_mgmt").getACall() and
+          DataFlow::localFlow(h, acctMgmtCall.getArg(0))
+        )
+      )
+    }
+  }
+}

python/ql/lib/semmle/python/security/dataflow/PamAuthorizationQuery.qll

@@ -0,0 +1,39 @@
+/**
+ * Provides a taint-tracking configuration for detecting "PAM Authorization" vulnerabilities.
+ *
+ * Note, for performance reasons: only import this file if
+ * `PamAuthorization::Configuration` is needed, otherwise
+ * `PamAuthorizationCustomizations` should be imported instead.
+ */
+
+import python
+import semmle.python.ApiGraphs
+import semmle.python.dataflow.new.TaintTracking
+import PamAuthorizationCustomizations::PamAuthorizationCustomizations
+
+/**
+ * A taint-tracking configuration for detecting "PAM Authorization" vulnerabilities.
+ */
+class Configuration extends TaintTracking::Configuration {
+  Configuration() { this = "PamAuthorization" }
+
+  override predicate isSource(DataFlow::Node node) { node instanceof Source }
+
+  override predicate isSink(DataFlow::Node node) { node instanceof Sink }
+
+  override predicate isAdditionalTaintStep(DataFlow::Node node1, DataFlow::Node node2) {
+    // Models flow from a remotely supplied username field to a PAM `handle`.
+    // `retval = pam_start(service, username, byref(conv), byref(handle))`
+    exists(API::CallNode pamStart, DataFlow::Node handle, API::CallNode pointer |
+      pointer = API::moduleImport("ctypes").getMember(["pointer", "byref"]).getACall() and
+      pamStart = libPam().getMember("pam_start").getACall() and
+      pointer = pamStart.getArg(3) and
+      handle = pointer.getArg(0) and
+      pamStart.getArg(1) = node1 and
+      handle = node2
+    )
+    or
+    // Flow from handle to the authenticate call in the final step
+    exists(VulnPamAuthCall c | c.getArg(0) = node1 | node2 = c)
+  }
+}

python/ql/lib/semmle/python/security/dataflow/StackTraceExposureCustomizations.qll

@@ -41,9 +41,7 @@ module StackTraceExposure {
   /**
    * A source of exception info, considered as a flow source.
    */
-  class ExceptionInfoAsSource extends Source {
-    ExceptionInfoAsSource() { this instanceof ExceptionInfo }
-  }
+  class ExceptionInfoAsSource extends Source instanceof ExceptionInfo { }
 
   /**
    * The body of a HTTP response that will be returned from a server, considered as a flow sink.

python/ql/lib/semmle/python/security/dataflow/XmlBombCustomizations.qll

@@ -30,9 +30,7 @@ module XmlBomb {
   abstract class Sanitizer extends DataFlow::Node { }
 
   /** A source of remote user input, considered as a flow source for XML bomb vulnerabilities. */
-  class RemoteFlowSourceAsSource extends Source {
-    RemoteFlowSourceAsSource() { this instanceof RemoteFlowSource }
-  }
+  class RemoteFlowSourceAsSource extends Source instanceof RemoteFlowSource { }
 
   /**
    * A call to an XML parser that is vulnerable to XML bombs.

python/ql/lib/semmle/python/security/dataflow/XxeCustomizations.qll

@@ -30,9 +30,7 @@ module Xxe {
   abstract class Sanitizer extends DataFlow::Node { }
 
   /** A source of remote user input, considered as a flow source for XXE vulnerabilities. */
-  class RemoteFlowSourceAsSource extends Source {
-    RemoteFlowSourceAsSource() { this instanceof RemoteFlowSource }
-  }
+  class RemoteFlowSourceAsSource extends Source instanceof RemoteFlowSource { }
 
   /**
    * A call to an XML parser that is vulnerable to XXE.

python/ql/lib/semmle/python/security/internal/EncryptionKeySizes.qll

@@ -0,0 +1,21 @@
+/**
+ * INTERNAL: Do not use.
+ *
+ * Provides predicates for recommended encryption key sizes.
+ * Such that we can share this logic across our CodeQL analysis of different languages.
+ */
+
+/** Returns the minimum recommended key size for RSA. */
+int minSecureKeySizeRsa() { result = 2048 }
+
+/** Returns the minimum recommended key size for DSA. */
+int minSecureKeySizeDsa() { result = 2048 }
+
+/** Returns the minimum recommended key size for DH. */
+int minSecureKeySizeDh() { result = 2048 }
+
+/** Returns the minimum recommended key size for elliptic curve cryptography. */
+int minSecureKeySizeEcc() { result = 256 }
+
+/** Returns the minimum recommended key size for AES. */
+int minSecureKeySizeAes() { result = 128 }

python/ql/lib/semmle/python/security/regexp/HostnameRegex.qll

@@ -0,0 +1,18 @@
+/**
+ * Provides predicates for reasoning about regular expressions
+ * that match URLs and hostname patterns.
+ */
+
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.RegexTreeView::RegexTreeView as TreeImpl
+private import semmle.python.dataflow.new.Regexp as Regexp
+private import codeql.regex.HostnameRegexp as Shared
+
+private module Impl implements Shared::HostnameRegexpSig<TreeImpl> {
+  class DataFlowNode = DataFlow::Node;
+
+  class RegExpPatternSource = Regexp::RegExpPatternSource;
+}
+
+import Shared::Make<TreeImpl, Impl>

python/ql/lib/semmle/python/types/Exceptions.qll

@@ -367,6 +367,10 @@ predicate scope_raises_unknown(Scope s) {
 class ExceptFlowNode extends ControlFlowNode {
   ExceptFlowNode() { this.getNode() instanceof ExceptStmt }
 
+  /**
+   * Gets the type handled by this exception handler.
+   * `ExceptionType` in `except ExceptionType as e:`
+   */
   ControlFlowNode getType() {
     exists(ExceptStmt ex |
       this.getBasicBlock().dominates(result.getBasicBlock()) and
@@ -375,6 +379,10 @@ class ExceptFlowNode extends ControlFlowNode {
     )
   }
 
+  /**
+   * Gets the name assigned to the handled exception, if any.
+   * `e` in `except ExceptionType as e:`
+   */
   ControlFlowNode getName() {
     exists(ExceptStmt ex |
       this.getBasicBlock().dominates(result.getBasicBlock()) and
@@ -439,6 +447,29 @@ class ExceptFlowNode extends ControlFlowNode {
   }
 }
 
+/** The ControlFlowNode for an 'except*' statement. */
+class ExceptGroupFlowNode extends ControlFlowNode {
+  ExceptGroupFlowNode() { this.getNode() instanceof ExceptGroupStmt }
+
+  /**
+   * Gets the type handled by this exception handler.
+   * `ExceptionType` in `except* ExceptionType as e:`
+   */
+  ControlFlowNode getType() {
+    this.getBasicBlock().dominates(result.getBasicBlock()) and
+    result = this.getNode().(ExceptGroupStmt).getType().getAFlowNode()
+  }
+
+  /**
+   * Gets the name assigned to the handled exception, if any.
+   * `e` in `except* ExceptionType as e:`
+   */
+  ControlFlowNode getName() {
+    this.getBasicBlock().dominates(result.getBasicBlock()) and
+    result = this.getNode().(ExceptGroupStmt).getName().getAFlowNode()
+  }
+}
+
 private ControlFlowNode element_from_tuple_objectapi(Object tuple) {
   exists(Tuple t | t = tuple.getOrigin() and result = t.getAnElt().getAFlowNode())
 }

python/ql/lib/semmle/python/types/FunctionObject.qll

@@ -10,7 +10,7 @@ private import semmle.python.types.Builtins
 abstract class FunctionObject extends Object {
   CallableValue theCallable() { result.(ObjectInternal).getSource() = this }
 
-  predicate isOverridingMethod() { exists(Object f | this.overrides(f)) }
+  predicate isOverridingMethod() { this.overrides(_) }
 
   predicate isOverriddenMethod() { exists(Object f | f.overrides(this)) }
 

python/ql/lib/semmle/python/web/turbogears/TurboGears.qll

@@ -28,10 +28,10 @@ deprecated class TurboGearsControllerMethod extends Function {
   predicate isTemplated() { exists(this.templateName()) }
 
   Dict getValidationDict() {
-    exists(Call call, Value dict |
+    exists(Call call |
       call = this.getADecorator() and
       call.getFunc().(Name).getId() = "validate" and
-      call.getArg(0).pointsTo(dict, result)
+      call.getArg(0).pointsTo(_, result)
     )
   }
 }

python/ql/lib/semmlecode.python.dbscheme

@@ -273,6 +273,11 @@ py_extracted_version(int module : @py_Module ref,
 /* <Field> Ellipsis.location = 0, location */
 /* <Field> Ellipsis.parenthesised = 1, bool */
 
+/* <Field> ExceptGroupStmt.location = 0, location */
+/* <Field> ExceptGroupStmt.type = 1, expr */
+/* <Field> ExceptGroupStmt.name = 2, expr */
+/* <Field> ExceptGroupStmt.body = 3, stmt_list */
+
 /* <Field> ExceptStmt.location = 0, location */
 /* <Field> ExceptStmt.type = 1, expr */
 /* <Field> ExceptStmt.name = 2, expr */
@@ -863,25 +868,26 @@ case @py_stmt.kind of
 |   4 = @py_Continue
 |   5 = @py_Delete
 |   6 = @py_ExceptStmt
-|   7 = @py_Exec
-|   8 = @py_Expr_stmt
-|   9 = @py_For
-|   10 = @py_Global
-|   11 = @py_If
-|   12 = @py_Import
-|   13 = @py_ImportStar
-|   14 = @py_MatchStmt
-|   15 = @py_Case
-|   16 = @py_Nonlocal
-|   17 = @py_Pass
-|   18 = @py_Print
-|   19 = @py_Raise
-|   20 = @py_Return
-|   21 = @py_Try
-|   22 = @py_While
-|   23 = @py_With
-|   24 = @py_TemplateWrite
-|   25 = @py_AnnAssign;
+|   7 = @py_ExceptGroupStmt
+|   8 = @py_Exec
+|   9 = @py_Expr_stmt
+|   10 = @py_For
+|   11 = @py_Global
+|   12 = @py_If
+|   13 = @py_Import
+|   14 = @py_ImportStar
+|   15 = @py_MatchStmt
+|   16 = @py_Case
+|   17 = @py_Nonlocal
+|   18 = @py_Pass
+|   19 = @py_Print
+|   20 = @py_Raise
+|   21 = @py_Return
+|   22 = @py_Try
+|   23 = @py_While
+|   24 = @py_With
+|   25 = @py_TemplateWrite
+|   26 = @py_AnnAssign;
 
 case @py_unaryop.kind of
     0 = @py_Invert
@@ -907,7 +913,7 @@ case @py_unaryop.kind of
 
 @py_expr_or_stmt = @py_expr | @py_stmt;
 
-@py_expr_parent = @py_AnnAssign | @py_Assert | @py_Assign | @py_AssignExpr | @py_Attribute | @py_AugAssign | @py_Await | @py_BinaryExpr | @py_Call | @py_Case | @py_Compare | @py_DictComp | @py_DictUnpacking | @py_ExceptStmt | @py_Exec | @py_Expr_stmt | @py_Filter | @py_For | @py_FormattedValue | @py_Function | @py_FunctionExpr | @py_GeneratorExp | @py_Guard | @py_If | @py_IfExp | @py_ImportMember | @py_ImportStar | @py_KeyValuePair | @py_ListComp | @py_MatchAsPattern | @py_MatchCapturePattern | @py_MatchClassPattern | @py_MatchKeywordPattern | @py_MatchLiteralPattern | @py_MatchStmt | @py_MatchValuePattern | @py_Print | @py_Raise | @py_Repr | @py_Return | @py_SetComp | @py_Slice | @py_Starred | @py_Subscript | @py_TemplateDottedNotation | @py_TemplateWrite | @py_UnaryExpr | @py_While | @py_With | @py_Yield | @py_YieldFrom | @py_alias | @py_arguments | @py_comprehension | @py_expr_list | @py_keyword | @py_parameter_list;
+@py_expr_parent = @py_AnnAssign | @py_Assert | @py_Assign | @py_AssignExpr | @py_Attribute | @py_AugAssign | @py_Await | @py_BinaryExpr | @py_Call | @py_Case | @py_Compare | @py_DictComp | @py_DictUnpacking | @py_ExceptGroupStmt | @py_ExceptStmt | @py_Exec | @py_Expr_stmt | @py_Filter | @py_For | @py_FormattedValue | @py_Function | @py_FunctionExpr | @py_GeneratorExp | @py_Guard | @py_If | @py_IfExp | @py_ImportMember | @py_ImportStar | @py_KeyValuePair | @py_ListComp | @py_MatchAsPattern | @py_MatchCapturePattern | @py_MatchClassPattern | @py_MatchKeywordPattern | @py_MatchLiteralPattern | @py_MatchStmt | @py_MatchValuePattern | @py_Print | @py_Raise | @py_Repr | @py_Return | @py_SetComp | @py_Slice | @py_Starred | @py_Subscript | @py_TemplateDottedNotation | @py_TemplateWrite | @py_UnaryExpr | @py_While | @py_With | @py_Yield | @py_YieldFrom | @py_alias | @py_arguments | @py_comprehension | @py_expr_list | @py_keyword | @py_parameter_list;
 
 @py_location_parent = @py_DictUnpacking | @py_KeyValuePair | @py_StringPart | @py_comprehension | @py_expr | @py_keyword | @py_pattern | @py_stmt;
 
@@ -919,7 +925,7 @@ case @py_unaryop.kind of
 
 @py_scope = @py_Class | @py_Function | @py_Module;
 
-@py_stmt_list_parent = @py_Case | @py_Class | @py_ExceptStmt | @py_For | @py_Function | @py_If | @py_MatchStmt | @py_Module | @py_Try | @py_While | @py_With;
+@py_stmt_list_parent = @py_Case | @py_Class | @py_ExceptGroupStmt | @py_ExceptStmt | @py_For | @py_Function | @py_If | @py_MatchStmt | @py_Module | @py_Try | @py_While | @py_With;
 
 @py_str_list_parent = @py_Global | @py_Nonlocal;
 

python/ql/lib/semmlecode.python.dbscheme.stats

@@ -469,6 +469,10 @@
 <v>5610</v>
 </e>
 <e>
+<k>@py_ExceptGroupStmt</k>
+<v>1000</v>
+</e>
+<e>
 <k>@py_Expr_stmt</k>
 <v>76750</v>
 </e>

python/ql/lib/upgrades/503c0516fba2e5da9570f00eb34ef43025ecb8fb/py_stmts.ql

@@ -0,0 +1,30 @@
+// First we need to wrap some database types
+class Stmt_ extends @py_stmt {
+  string toString() { result = "Stmt" }
+}
+
+class StmtList_ extends @py_stmt_list {
+  string toString() { result = "StmtList" }
+}
+
+/**
+ * New kinds have been inserted such that
+ * `@py_Exec` which used to have index 7 now has index 8.
+ * Entries with lower indices are unchanged.
+ */
+bindingset[old_index]
+int new_index(int old_index) {
+  if old_index < 7 then result = old_index else result = (8 - 7) + old_index
+}
+
+// The schema for py_stmts is:
+//
+// py_stmts(unique int id : @py_stmt,
+//   int kind: int ref,
+//   int parent : @py_stmt_list ref,
+//   int idx : int ref);
+from Stmt_ expr, int old_kind, StmtList_ parent, int idx, int new_kind
+where
+  py_stmts(expr, old_kind, parent, idx) and
+  new_kind = new_index(old_kind)
+select expr, new_kind, parent, idx

python/ql/lib/upgrades/503c0516fba2e5da9570f00eb34ef43025ecb8fb/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add support for `except*`
+compatibility: backwards
+py_stmts.rel: run py_stmts.qlo

python/ql/src/AlertSuppression.ql

@@ -0,0 +1,51 @@
+/**
+ * @name Alert suppression
+ * @description Generates information about alert suppressions.
+ * @kind alert-suppression
+ * @id py/alert-suppression
+ */
+
+private import codeql.util.suppression.AlertSuppression as AS
+private import semmle.python.Comment as P
+
+class AstNode instanceof P::AstNode {
+  predicate hasLocationInfo(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  ) {
+    super.getLocation().hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+  }
+
+  string toString() { result = super.toString() }
+}
+
+class SingleLineComment instanceof P::Comment {
+  predicate hasLocationInfo(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  ) {
+    super.getLocation().hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+  }
+
+  string getText() { result = super.getContents() }
+
+  string toString() { result = super.toString() }
+}
+
+import AS::Make<AstNode, SingleLineComment>
+
+/**
+ * A noqa suppression comment. Both pylint and pyflakes respect this, so lgtm ought to too.
+ */
+class NoqaSuppressionComment extends SuppressionComment instanceof SingleLineComment {
+  NoqaSuppressionComment() {
+    SingleLineComment.super.getText().regexpMatch("(?i)\\s*noqa\\s*([^:].*)?")
+  }
+
+  override string getAnnotation() { result = "lgtm" }
+
+  override predicate covers(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  ) {
+    this.hasLocationInfo(filepath, startline, _, endline, endcolumn) and
+    startcolumn = 1
+  }
+}

python/ql/src/CHANGELOG.md

@@ -1,3 +1,23 @@
+## 0.6.1
+
+No user-facing changes.
+
+## 0.6.0
+
+### Minor Analysis Improvements
+
+* The `analysis/AlertSuppression.ql` query has moved to the root folder. Users that refer to this query by path should update their configurations. The query has been updated to support the new `# codeql[query-id]` supression comments. These comments can be used to suppress an alert and must be placed on a blank line before the alert. In addition the legacy `# lgtm` and `# lgtm[query-id]` comments can now also be placed on the line before an alert.
+* Bumped the minimum keysize we consider secure for elliptic curve cryptography from 224 to 256 bits, following current best practices. This might effect results from the _Use of weak cryptographic key_ (`py/weak-crypto-key`) query.
+* Added modeling of `getpass.getpass` as a source of passwords, which will be an additional source for `py/clear-text-logging-sensitive-data`, `py/clear-text-storage-sensitive-data`, and `py/weak-sensitive-data-hashing`.
+
+## 0.5.6
+
+No user-facing changes.
+
+## 0.5.5
+
+No user-facing changes.
+
 ## 0.5.4
 
 No user-facing changes.

python/ql/src/Expressions/WrongNumberArgumentsForFormat.ql

@@ -17,15 +17,15 @@ import semmle.python.strings
 
 predicate string_format(BinaryExpr operation, StrConst str, Value args, AstNode origin) {
   operation.getOp() instanceof Mod and
-  exists(Value fmt, Context ctx |
-    operation.getLeft().pointsTo(ctx, fmt, str) and
+  exists(Context ctx |
+    operation.getLeft().pointsTo(ctx, _, str) and
     operation.getRight().pointsTo(ctx, args, origin)
   )
 }
 
 int sequence_length(Value args) {
   /* Guess length of sequence */
-  exists(Tuple seq, AstNode origin | seq.pointsTo(args, origin) |
+  exists(Tuple seq | seq.pointsTo(args, _) |
     result = strictcount(seq.getAnElt()) and
     not seq.getAnElt() instanceof Starred
   )

python/ql/src/Security/CWE-020-ExternalAPIs/ExternalAPIs.qll

@@ -87,12 +87,12 @@ class ExternalApiDataNode extends DataFlow::Node {
       not exists(cv.(CallableValue).getScope().getLocation().getFile().getRelativePath())
     ) and
     // Not already modeled as a taint step
-    not exists(DataFlow::Node next | TaintTrackingPrivate::defaultAdditionalTaintStep(this, next)) and
+    not TaintTrackingPrivate::defaultAdditionalTaintStep(this, _) and
     // for `list.append(x)`, we have a additional taint step from x -> [post] list.
     // Since we have modeled this explicitly, I don't see any cases where we would want to report this.
-    not exists(DataFlow::Node prev, DataFlow::PostUpdateNode post |
+    not exists(DataFlow::PostUpdateNode post |
       post.getPreUpdateNode() = this and
-      TaintTrackingPrivate::defaultAdditionalTaintStep(prev, post)
+      TaintTrackingPrivate::defaultAdditionalTaintStep(_, post)
     )
   }
 

python/ql/src/Security/CWE-020/HostnameRegexpShared.qll

@@ -3,200 +3,6 @@
  * that match URLs and hostname patterns.
  */
 
-private import HostnameRegexpSpecific
-
-/**
- * Holds if the given constant is unlikely to occur in the origin part of a URL.
- */
-predicate isConstantInvalidInsideOrigin(RegExpConstant term) {
-  // Look for any of these cases:
-  // - A character that can't occur in the origin
-  // - Two dashes in a row
-  // - A colon that is not part of port or scheme separator
-  // - A slash that is not part of scheme separator
-  term.getValue().regexpMatch(".*(?:[^a-zA-Z0-9.:/-]|--|:[^0-9/]|(?<![/:]|^)/).*")
-}
-
-/** Holds if `term` is a dot constant of form `\.` or `[.]`. */
-predicate isDotConstant(RegExpTerm term) {
-  term.(RegExpCharEscape).getValue() = "."
-  or
-  exists(RegExpCharacterClass cls |
-    term = cls and
-    not cls.isInverted() and
-    cls.getNumChild() = 1 and
-    cls.getAChild().(RegExpConstant).getValue() = "."
-  )
-}
-
-/** Holds if `term` is a wildcard `.` or an actual `.` character. */
-predicate isDotLike(RegExpTerm term) {
-  term instanceof RegExpDot
-  or
-  isDotConstant(term)
-}
-
-/** Holds if `term` will only ever be matched against the beginning of the input. */
-predicate matchesBeginningOfString(RegExpTerm term) {
-  term.isRootTerm()
-  or
-  exists(RegExpTerm parent | matchesBeginningOfString(parent) |
-    term = parent.(RegExpSequence).getChild(0)
-    or
-    parent.(RegExpSequence).getChild(0) instanceof RegExpCaret and
-    term = parent.(RegExpSequence).getChild(1)
-    or
-    term = parent.(RegExpAlt).getAChild()
-    or
-    term = parent.(RegExpGroup).getAChild()
-  )
-}
-
-/**
- * Holds if the given sequence `seq` contains top-level domain preceded by a dot, such as `.com`,
- * excluding cases where this is at the very beginning of the regexp.
- *
- * `i` is bound to the index of the last child in the top-level domain part.
- */
-predicate hasTopLevelDomainEnding(RegExpSequence seq, int i) {
-  seq.getChild(i)
-      .(RegExpConstant)
-      .getValue()
-      .regexpMatch("(?i)" + RegExpPatterns::getACommonTld() + "(:\\d+)?([/?#].*)?") and
-  isDotLike(seq.getChild(i - 1)) and
-  not (i = 1 and matchesBeginningOfString(seq))
-}
-
-/**
- * Holds if the given regular expression term contains top-level domain preceded by a dot,
- * such as `.com`.
- */
-predicate hasTopLevelDomainEnding(RegExpSequence seq) { hasTopLevelDomainEnding(seq, _) }
-
-/**
- * Holds if `term` will always match a hostname, that is, all disjunctions contain
- * a hostname pattern that isn't inside a quantifier.
- */
-predicate alwaysMatchesHostname(RegExpTerm term) {
-  hasTopLevelDomainEnding(term, _)
-  or
-  // `localhost` is considered a hostname pattern, but has no TLD
-  term.(RegExpConstant).getValue().regexpMatch("\\blocalhost\\b")
-  or
-  not term instanceof RegExpAlt and
-  not term instanceof RegExpQuantifier and
-  alwaysMatchesHostname(term.getAChild())
-  or
-  alwaysMatchesHostnameAlt(term)
-}
-
-/** Holds if every child of `alt` contains a hostname pattern. */
-predicate alwaysMatchesHostnameAlt(RegExpAlt alt) {
-  alwaysMatchesHostnameAlt(alt, alt.getNumChild() - 1)
-}
-
-/**
- * Holds if the first `i` children of `alt` contains a hostname pattern.
- *
- * This is used instead of `forall` to avoid materializing the set of alternatives
- * that don't contains hostnames, which is much larger.
- */
-predicate alwaysMatchesHostnameAlt(RegExpAlt alt, int i) {
-  alwaysMatchesHostname(alt.getChild(0)) and i = 0
-  or
-  alwaysMatchesHostnameAlt(alt, i - 1) and
-  alwaysMatchesHostname(alt.getChild(i))
-}
-
-/**
- * Holds if `term` occurs inside a quantifier or alternative (and thus
- * can not be expected to correspond to a unique match), or as part of
- * a lookaround assertion (which are rarely used for capture groups).
- */
-predicate isInsideChoiceOrSubPattern(RegExpTerm term) {
-  exists(RegExpParent parent | parent = term.getParent() |
-    parent instanceof RegExpAlt
-    or
-    parent instanceof RegExpQuantifier
-    or
-    parent instanceof RegExpSubPattern
-    or
-    isInsideChoiceOrSubPattern(parent)
-  )
-}
-
-/**
- * Holds if `group` is likely to be used as a capture group.
- */
-predicate isLikelyCaptureGroup(RegExpGroup group) {
-  group.isCapture() and
-  not isInsideChoiceOrSubPattern(group)
-}
-
-/**
- * Holds if `seq` contains two consecutive dots `..` or escaped dots.
- *
- * At least one of these dots is not intended to be a subdomain separator,
- * so we avoid flagging the pattern in this case.
- */
-predicate hasConsecutiveDots(RegExpSequence seq) {
-  exists(int i |
-    isDotLike(seq.getChild(i)) and
-    isDotLike(seq.getChild(i + 1))
-  )
-}
-
-predicate isIncompleteHostNameRegExpPattern(RegExpTerm regexp, RegExpSequence seq, string msg) {
-  seq = regexp.getAChild*() and
-  exists(RegExpDot unescapedDot, int i, string hostname |
-    hasTopLevelDomainEnding(seq, i) and
-    not isConstantInvalidInsideOrigin(seq.getChild([0 .. i - 1]).getAChild*()) and
-    not isLikelyCaptureGroup(seq.getChild([i .. seq.getNumChild() - 1]).getAChild*()) and
-    unescapedDot = seq.getChild([0 .. i - 1]).getAChild*() and
-    unescapedDot != seq.getChild(i - 1) and // Should not be the '.' immediately before the TLD
-    not hasConsecutiveDots(unescapedDot.getParent()) and
-    hostname =
-      seq.getChild(i - 2).getRawValue() + seq.getChild(i - 1).getRawValue() +
-        seq.getChild(i).getRawValue()
-  |
-    if unescapedDot.getParent() instanceof RegExpQuantifier
-    then
-      // `.*\.example.com` can match `evil.com/?x=.example.com`
-      //
-      // This problem only occurs when the pattern is applied against a full URL, not just a hostname/origin.
-      // We therefore check if the pattern includes a suffix after the TLD, such as `.*\.example.com/`.
-      // Note that a post-anchored pattern (`.*\.example.com$`) will usually fail to match a full URL,
-      // and patterns with neither a suffix nor an anchor fall under the purview of MissingRegExpAnchor.
-      seq.getChild(0) instanceof RegExpCaret and
-      not seq.getAChild() instanceof RegExpDollar and
-      seq.getChild([i .. i + 1]).(RegExpConstant).getValue().regexpMatch(".*[/?#].*") and
-      msg =
-        "has an unrestricted wildcard '" + unescapedDot.getParent().(RegExpQuantifier).getRawValue()
-          + "' which may cause '" + hostname +
-          "' to be matched anywhere in the URL, outside the hostname."
-    else
-      msg =
-        "has an unescaped '.' before '" + hostname +
-          "', so it might match more hosts than expected."
-  )
-}
-
-predicate incompleteHostnameRegExp(
-  RegExpSequence hostSequence, string message, DataFlow::Node aux, string label
-) {
-  exists(RegExpPatternSource re, RegExpTerm regexp, string msg, string kind |
-    regexp = re.getRegExpTerm() and
-    isIncompleteHostNameRegExpPattern(regexp, hostSequence, msg) and
-    (
-      if re.getAParse() != re
-      then (
-        kind = "string, which is used as a regular expression $@," and
-        aux = re.getAParse()
-      ) else (
-        kind = "regular expression" and aux = re
-      )
-    )
-  |
-    message = "This " + kind + " " + msg and label = "here"
-  )
-}
+// HostnameRegexp should be used directly from the shared regex pack, and not from this file.
+deprecated private import semmle.python.security.regexp.HostnameRegex as Dep
+import Dep

python/ql/src/Security/CWE-020/HostnameRegexpSpecific.qll

@@ -1,3 +0,0 @@
-import semmle.python.RegexTreeView
-import semmle.python.dataflow.new.DataFlow
-import semmle.python.dataflow.new.Regexp

python/ql/src/Security/CWE-020/IncompleteHostnameRegExp.ql

@@ -11,6 +11,6 @@
  *       external/cwe/cwe-020
  */
 
-import HostnameRegexpShared
+private import semmle.python.security.regexp.HostnameRegex as HostnameRegex
 
-query predicate problems = incompleteHostnameRegExp/4;
+query predicate problems = HostnameRegex::incompleteHostnameRegExp/4;

python/ql/src/Security/CWE-285/PamAuthorization.ql

@@ -1,7 +1,7 @@
 /**
  * @name PAM authorization bypass due to incorrect usage
  * @description Not using `pam_acct_mgmt` after `pam_authenticate` to check the validity of a login can lead to authorization bypass.
- * @kind problem
+ * @kind path-problem
  * @problem.severity warning
  * @security-severity 8.1
  * @precision high
@@ -11,28 +11,12 @@
  */
 
 import python
+import DataFlow::PathGraph
 import semmle.python.ApiGraphs
-import experimental.semmle.python.Concepts
-import semmle.python.dataflow.new.TaintTracking
+import semmle.python.security.dataflow.PamAuthorizationQuery
 
-API::Node libPam() {
-  exists(API::CallNode findLibCall, API::CallNode cdllCall |
-    findLibCall = API::moduleImport("ctypes").getMember("util").getMember("find_library").getACall() and
-    findLibCall.getParameter(0).getAValueReachingSink().asExpr().(StrConst).getText() = "pam" and
-    cdllCall = API::moduleImport("ctypes").getMember("CDLL").getACall() and
-    cdllCall.getParameter(0).getAValueReachingSink() = findLibCall
-  |
-    result = cdllCall.getReturn()
-  )
-}
-
-from API::CallNode authenticateCall, DataFlow::Node handle
-where
-  authenticateCall = libPam().getMember("pam_authenticate").getACall() and
-  handle = authenticateCall.getArg(0) and
-  not exists(API::CallNode acctMgmtCall |
-    acctMgmtCall = libPam().getMember("pam_acct_mgmt").getACall() and
-    DataFlow::localFlow(handle, acctMgmtCall.getArg(0))
-  )
-select authenticateCall,
-  "This PAM authentication call may lead to an authorization bypass, since 'pam_acct_mgmt' is not called afterwards."
+from Configuration config, DataFlow::PathNode source, DataFlow::PathNode sink
+where config.hasFlowPath(source, sink)
+select sink.getNode(), source, sink,
+  "This PAM authentication depends on a $@, and 'pam_acct_mgmt' is not called afterwards.",
+  source.getNode(), "user-provided value"

python/ql/src/Security/CWE-326/WeakCryptoKey.qhelp

@@ -11,13 +11,13 @@ As computational power increases, the ability to break ciphers grows and keys ne
 <p>
 The three main asymmetric key algorithms currently in use are Rivest–Shamir–Adleman (RSA) cryptography, Digital Signature Algorithm (DSA), and Elliptic-curve cryptography (ECC).
 With current technology, key sizes of 2048 bits for RSA and DSA,
-or 224 bits for ECC, are regarded as unbreakable.
+or 256 bits for ECC, are regarded as unbreakable.
 </p>
 </overview>
 
 <recommendation>
 <p>
-Increase the key size to the recommended amount or larger. For RSA or DSA this is at least 2048 bits, for ECC this is at least 224 bits.
+Increase the key size to the recommended amount or larger. For RSA or DSA this is at least 2048 bits, for ECC this is at least 256 bits.
 </p>
 </recommendation>
 
@@ -45,4 +45,3 @@ Recommendation for Transitioning the Use of Cryptographic Algorithms and Key Len
 </li>
 </references>
 </qhelp>
-

python/ql/src/Statements/StatementNoEffect.ql

@@ -22,8 +22,8 @@ predicate understood_attribute(Attribute attr, ClassValue cls, ClassValue attr_c
 
 /* Conservative estimate of whether attribute lookup has a side effect */
 predicate side_effecting_attribute(Attribute attr) {
-  exists(ClassValue cls, ClassValue attr_cls |
-    understood_attribute(attr, cls, attr_cls) and
+  exists(ClassValue attr_cls |
+    understood_attribute(attr, _, attr_cls) and
     side_effecting_descriptor_type(attr_cls)
   )
 }

python/ql/src/analysis/AlertSuppression.ql

@@ -1,107 +0,0 @@
-/**
- * @name Alert suppression
- * @description Generates information about alert suppressions.
- * @kind alert-suppression
- * @id py/alert-suppression
- */
-
-import python
-
-/**
- * An alert suppression comment.
- */
-abstract class SuppressionComment extends Comment {
-  /** Gets the scope of this suppression. */
-  abstract SuppressionScope getScope();
-
-  /** Gets the suppression annotation in this comment. */
-  abstract string getAnnotation();
-
-  /**
-   * Holds if this comment applies to the range from column `startcolumn` of line `startline`
-   * to column `endcolumn` of line `endline` in file `filepath`.
-   */
-  abstract predicate covers(
-    string filepath, int startline, int startcolumn, int endline, int endcolumn
-  );
-}
-
-/**
- * An alert comment that applies to a single line
- */
-abstract class LineSuppressionComment extends SuppressionComment {
-  LineSuppressionComment() {
-    exists(string filepath, int l |
-      this.getLocation().hasLocationInfo(filepath, l, _, _, _) and
-      any(AstNode a).getLocation().hasLocationInfo(filepath, l, _, _, _)
-    )
-  }
-
-  /** Gets the scope of this suppression. */
-  override SuppressionScope getScope() { result = this }
-
-  override predicate covers(
-    string filepath, int startline, int startcolumn, int endline, int endcolumn
-  ) {
-    this.getLocation().hasLocationInfo(filepath, startline, _, endline, endcolumn) and
-    startcolumn = 1
-  }
-}
-
-/**
- * An lgtm suppression comment.
- */
-class LgtmSuppressionComment extends LineSuppressionComment {
-  string annotation;
-
-  LgtmSuppressionComment() {
-    exists(string all | all = this.getContents() |
-      // match `lgtm[...]` anywhere in the comment
-      annotation = all.regexpFind("(?i)\\blgtm\\s*\\[[^\\]]*\\]", _, _)
-      or
-      // match `lgtm` at the start of the comment and after semicolon
-      annotation = all.regexpFind("(?i)(?<=^|;)\\s*lgtm(?!\\B|\\s*\\[)", _, _).trim()
-    )
-  }
-
-  /** Gets the suppression annotation in this comment. */
-  override string getAnnotation() { result = annotation }
-}
-
-/**
- * A noqa suppression comment. Both pylint and pyflakes respect this, so lgtm ought to too.
- */
-class NoqaSuppressionComment extends LineSuppressionComment {
-  NoqaSuppressionComment() { this.getContents().toLowerCase().regexpMatch("\\s*noqa\\s*([^:].*)?") }
-
-  override string getAnnotation() { result = "lgtm" }
-}
-
-/**
- * The scope of an alert suppression comment.
- */
-class SuppressionScope extends @py_comment {
-  SuppressionScope() { this instanceof SuppressionComment }
-
-  /**
-   * Holds if this element is at the specified location.
-   * The location spans column `startcolumn` of line `startline` to
-   * column `endcolumn` of line `endline` in file `filepath`.
-   * For more information, see
-   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
-   */
-  predicate hasLocationInfo(
-    string filepath, int startline, int startcolumn, int endline, int endcolumn
-  ) {
-    this.(SuppressionComment).covers(filepath, startline, startcolumn, endline, endcolumn)
-  }
-
-  /** Gets a textual representation of this element. */
-  string toString() { result = "suppression range" }
-}
-
-from SuppressionComment c
-select c, // suppression comment
-  c.getContents(), // text of suppression comment (excluding delimiters)
-  c.getAnnotation(), // text of suppression annotation
-  c.getScope() // scope of suppression

python/ql/src/analysis/Consistency.ql

@@ -231,8 +231,8 @@ predicate points_to_consistency(string clsname, string problem, string what) {
     what = obj.toString()
   )
   or
-  exists(ControlFlowNode use, ControlFlowNode inter, Object obj |
-    intermediate_origins(use, inter, obj) and
+  exists(ControlFlowNode use, ControlFlowNode inter |
+    intermediate_origins(use, inter, _) and
     clsname = use.getAQlClass() and
     problem = "has intermediate origin " + inter and
     what = use.toString()

python/ql/src/analysis/TypeInferenceFailure.ql

@@ -12,5 +12,5 @@ import python
 from ControlFlowNode f, Object o
 where
   f.refersTo(o) and
-  not exists(ClassObject c | f.refersTo(o, c, _))
+  not f.refersTo(o, _, _)
 select o, "Type inference fails for 'object'."

python/ql/src/change-notes/2022-11-22-getpass.md

@@ -1,4 +0,0 @@
----
- category: minorAnalysis
----
- * Added modeling of `getpass.getpass` as a source of passwords, which will be an additional source for `py/clear-text-logging-sensitive-data`, `py/clear-text-storage-sensitive-data`, and `py/weak-sensitive-data-hashing`.

python/ql/src/change-notes/released/0.5.5.md

@@ -0,0 +1,3 @@
+## 0.5.5
+
+No user-facing changes.

python/ql/src/change-notes/released/0.5.6.md

@@ -0,0 +1,3 @@
+## 0.5.6
+
+No user-facing changes.

python/ql/src/change-notes/released/0.6.0.md

@@ -0,0 +1,7 @@
+## 0.6.0
+
+### Minor Analysis Improvements
+
+* The `analysis/AlertSuppression.ql` query has moved to the root folder. Users that refer to this query by path should update their configurations. The query has been updated to support the new `# codeql[query-id]` supression comments. These comments can be used to suppress an alert and must be placed on a blank line before the alert. In addition the legacy `# lgtm` and `# lgtm[query-id]` comments can now also be placed on the line before an alert.
+* Bumped the minimum keysize we consider secure for elliptic curve cryptography from 224 to 256 bits, following current best practices. This might effect results from the _Use of weak cryptographic key_ (`py/weak-crypto-key`) query.
+* Added modeling of `getpass.getpass` as a source of passwords, which will be an additional source for `py/clear-text-logging-sensitive-data`, `py/clear-text-storage-sensitive-data`, and `py/weak-sensitive-data-hashing`.

python/ql/src/change-notes/released/0.6.1.md

@@ -0,0 +1,3 @@
+## 0.6.1
+
+No user-facing changes.

python/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.5.4
+lastReleaseVersion: 0.6.1

python/ql/src/codeql-suites/python-security-experimental.qls

@@ -0,0 +1,4 @@
+- description: Extended and experimental security queries for Python
+- queries: .
+- apply: security-experimental-selectors.yml
+  from: codeql/suite-helpers

python/ql/src/experimental/Security/CWE-022/ZipSlip.ql

@@ -9,6 +9,7 @@
  * @security-severity 7.5
  * @precision high
  * @tags security
+ *       experimental
  *       external/cwe/cwe-022
  */
 

python/ql/src/experimental/Security/CWE-022bis/TarSlipImprov.qhelp.disabled

@@ -1,6 +1,7 @@
 <!DOCTYPE qhelp PUBLIC
   "-//Semmle//qhelp//EN"
   "qhelp.dtd">
+<!-- Disabled since it refers to examples which do not exist. -->
 <qhelp>
 
 <overview>

python/ql/src/experimental/Security/CWE-022bis/TarSlipImprov.ql

@@ -4,11 +4,12 @@
  *              destination file path is within the destination directory can cause files outside
  *              the destination directory to be overwritten.
  * @kind path-problem
- * @id py/tarslip
+ * @id py/tarslip-extended
  * @problem.severity error
  * @security-severity 7.5
  * @precision high
  * @tags security
+ *       experimental
  *       external/cwe/cwe-022
  */
 
@@ -100,19 +101,15 @@ class Configuration extends TaintTracking::Configuration {
   }
 
   override predicate isAdditionalTaintStep(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
-    exists(AttrRead attr, MethodCallNode call |
-      attr.accesses(nodeFrom, "getmembers") and
-      nodeFrom = call.getObject() and
-      nodeFrom instanceof AllTarfileOpens and
-      nodeTo = call
-    )
+    nodeTo.(MethodCallNode).calls(nodeFrom, "getmembers") and
+    nodeFrom instanceof AllTarfileOpens
     or
-    exists(API::CallNode closing |
-      closing = API::moduleImport("contextlib").getMember("closing").getACall() and
-      nodeFrom = closing.getArg(0) and
-      nodeFrom = tarfileOpen().getReturn().getAValueReachingSink() and
-      nodeTo = closing
-    )
+    // To handle the case of `with closing(tarfile.open()) as file:`
+    // we add a step from the first argument of `closing` to the call to `closing`,
+    // whenever that first argument is a return of `tarfile.open()`.
+    nodeTo = API::moduleImport("contextlib").getMember("closing").getACall() and
+    nodeFrom = nodeTo.(API::CallNode).getArg(0) and
+    nodeFrom = tarfileOpen().getReturn().getAValueReachableFromSource()
   }
 }
 

python/ql/src/experimental/Security/CWE-074/TemplateInjection.ql

@@ -6,6 +6,7 @@
  * @precision high
  * @id py/template-injection
  * @tags security
+ *       experimental
  *       external/cwe/cwe-074
  */
 

python/ql/src/experimental/Security/CWE-079/ReflectedXSS.ql

@@ -6,8 +6,9 @@
  * @problem.severity error
  * @security-severity 2.9
  * @sub-severity high
- * @id py/reflective-xss
+ * @id py/reflective-xss-email
  * @tags security
+ *       experimental
  *       external/cwe/cwe-079
  *       external/cwe/cwe-116
  */

python/ql/src/experimental/Security/CWE-091/Xslt.ql

@@ -7,6 +7,7 @@
  * @precision high
  * @id py/xslt-injection
  * @tags security
+ *       experimental
  *       external/cwe/cwe-643
  */
 

python/ql/src/experimental/Security/CWE-113/HeaderInjection.ql

@@ -6,6 +6,7 @@
  * @problem.severity error
  * @id py/header-injection
  * @tags security
+ *       experimental
  *       external/cwe/cwe-113
  *       external/cwe/cwe-079
  */

python/ql/src/experimental/Security/CWE-1236/CsvInjection.ql

@@ -6,6 +6,7 @@
  * @problem.severity error
  * @id py/csv-injection
  * @tags security
+ *       experimental
  *       external/cwe/cwe-1236
  */
 

python/ql/src/experimental/Security/CWE-287/ImproperLdapAuth.ql

@@ -5,6 +5,7 @@
  * @problem.severity warning
  * @id py/improper-ldap-auth
  * @tags security
+ *       experimental
  *       external/cwe/cwe-287
  */
 

python/ql/src/experimental/Security/CWE-327/Azure/UnsafeUsageOfClientSideEncryptionVersion.ql

@@ -3,6 +3,7 @@
  * @description Using version v1 of Azure Storage client-side encryption is insecure, and may enable an attacker to decrypt encrypted data
  * @kind problem
  * @tags security
+ *       experimental
  *       cryptography
  *       external/cwe/cwe-327
  * @id py/azure-storage/unsafe-client-side-encryption-in-use

python/ql/src/experimental/Security/CWE-338/InsecureRandomness.ql

@@ -9,6 +9,7 @@
  * @precision high
  * @id py/insecure-randomness
  * @tags security
+ *       experimental
  *       external/cwe/cwe-338
  */
 

python/ql/src/experimental/Security/CWE-340/TokenBuiltFromUUID.ql

@@ -8,6 +8,7 @@
  * @security-severity 5
  * @id py/predictable-token
  * @tags security
+ *       experimental
  *       external/cwe/cwe-340
  */
 

python/ql/src/experimental/Security/CWE-347/JWTEmptyKeyOrAlgorithm.ql

@@ -5,6 +5,7 @@
  * @problem.severity warning
  * @id py/jwt-empty-secret-or-algorithm
  * @tags security
+ *       experimental
  */
 
 // determine precision above

python/ql/src/experimental/Security/CWE-347/JWTMissingSecretOrPublicKeyVerification.ql

@@ -5,6 +5,7 @@
  * @problem.severity warning
  * @id py/jwt-missing-verification
  * @tags security
+ *       experimental
  *       external/cwe/cwe-347
  */
 

python/ql/src/experimental/Security/CWE-348/ClientSuppliedIpUsedInSecurityCheck.ql

@@ -7,6 +7,7 @@
  * @precision high
  * @id py/ip-address-spoofing
  * @tags security
+ *       experimental
  *       external/cwe/cwe-348
  */
 

python/ql/src/experimental/Security/CWE-522/LDAPInsecureAuth.ql

@@ -5,6 +5,7 @@
  * @problem.severity error
  * @id py/insecure-ldap-auth
  * @tags security
+ *       experimental
  *       external/cwe/cwe-522
  *       external/cwe/cwe-523
  */