Merge branch 'main' into tutorial/library-pack

python/ql/lib/change-notes/2022-11-14-grouped-exceptions.md

@@ -0,0 +1,5 @@
+---
+category: fix
+---
+* `except*` is now supported.
+* The result of `Try.getAHandler` and `Try.getHandler(<index>)` is no longer of type `ExceptStmt`, as handlers may also be `ExceptGroupStmt`s (After Python 3.11 introduced PEP 654). Instead, it is of the new type `ExceptionHandler` of which `ExceptStmt` and `ExceptGroupStmt` are subtypes. To support selecting only one type of handler, `Try.getANormalHandler` and `Try.getAGroupHandler` have been added. Existing uses of `Try.getAHandler` for which it is important to select only normal handlers, will need to be updated to `Try.getANormalHandler`.

python/ql/lib/semmle/python/AstGenerated.qll

@@ -389,6 +389,26 @@ class Eq_ extends @py_Eq, Cmpop {
   override string toString() { result = "Eq" }
 }
 
+/** INTERNAL: See the class `ExceptGroupStmt` for further information. */
+class ExceptGroupStmt_ extends @py_ExceptGroupStmt, Stmt {
+  /** Gets the type of this except group block. */
+  Expr getType() { py_exprs(result, _, this, 1) }
+
+  /** Gets the name of this except group block. */
+  Expr getName() { py_exprs(result, _, this, 2) }
+
+  /** Gets the body of this except group block. */
+  StmtList getBody() { py_stmt_lists(result, this, 3) }
+
+  /** Gets the nth statement of this except group block. */
+  Stmt getStmt(int index) { result = this.getBody().getItem(index) }
+
+  /** Gets a statement of this except group block. */
+  Stmt getAStmt() { result = this.getBody().getAnItem() }
+
+  override string toString() { result = "ExceptGroupStmt" }
+}
+
 /** INTERNAL: See the class `ExceptStmt` for further information. */
 class ExceptStmt_ extends @py_ExceptStmt, Stmt {
   /** Gets the type of this except block. */

python/ql/lib/semmle/python/Stmts.qll

@@ -143,12 +143,30 @@ class Exec extends Exec_ {
   override Stmt getASubStatement() { none() }
 }
 
-/** An except statement (part of a `try` statement), such as `except IOError as err:` */
-class ExceptStmt extends ExceptStmt_ {
-  /* syntax: except Expr [ as Expr ]: */
+/**
+ * An exception handler such as an `except` or an `except*` statement
+ * in a `try` statement.
+ */
+class ExceptionHandler extends Stmt {
+  ExceptionHandler() {
+    this instanceof ExceptStmt_
+    or
+    this instanceof ExceptGroupStmt_
+  }
+
   /** Gets the immediately enclosing try statement */
   Try getTry() { result.getAHandler() = this }
 
+  /** Gets the name of this except group block. */
+  abstract Expr getName();
+
+  /** Gets the type of this except group block. */
+  abstract Expr getType();
+}
+
+/** An except group statement (part of a `try` statement), such as `except* IOError as err:` */
+class ExceptGroupStmt extends ExceptGroupStmt_, ExceptionHandler {
+  /* syntax: except Expr [ as Expr ]: */
   override Expr getASubExpression() {
     result = this.getName()
     or
@@ -159,10 +177,34 @@ class ExceptStmt extends ExceptStmt_ {
 
   override Stmt getLastStatement() { result = this.getBody().getLastItem().getLastStatement() }
 
+  override Expr getName() { result = ExceptGroupStmt_.super.getName() }
+
   override Expr getType() {
-    result = super.getType() and not result instanceof Tuple
+    result = ExceptGroupStmt_.super.getType() and not result instanceof Tuple
     or
-    result = super.getType().(Tuple).getAnElt()
+    result = ExceptGroupStmt_.super.getType().(Tuple).getAnElt()
+  }
+}
+
+/** An except statement (part of a `try` statement), such as `except IOError as err:` */
+class ExceptStmt extends ExceptStmt_, ExceptionHandler {
+  /* syntax: except Expr [ as Expr ]: */
+  override Expr getASubExpression() {
+    result = this.getName()
+    or
+    result = this.getType()
+  }
+
+  override Stmt getASubStatement() { result = this.getAStmt() }
+
+  override Stmt getLastStatement() { result = this.getBody().getLastItem().getLastStatement() }
+
+  override Expr getName() { result = ExceptStmt_.super.getName() }
+
+  override Expr getType() {
+    result = ExceptStmt_.super.getType() and not result instanceof Tuple
+    or
+    result = ExceptStmt_.super.getType().(Tuple).getAnElt()
   }
 }
 
@@ -364,10 +406,15 @@ class Try extends Try_ {
     result = this.getAnOrelse()
   }
 
-  override ExceptStmt getHandler(int i) { result = Try_.super.getHandler(i) }
+  override ExceptionHandler getHandler(int i) { result = Try_.super.getHandler(i) }
 
-  /** Gets an exception handler of this try statement. */
-  override ExceptStmt getAHandler() { result = Try_.super.getAHandler() }
+  override ExceptionHandler getAHandler() { result = Try_.super.getAHandler() }
+
+  /** Gets a normal exception handler, `except`, of this try statement. */
+  ExceptStmt getANormalHandler() { result = this.getAHandler() }
+
+  /** Gets a group exception handler, `except*`, of this try statement. */
+  ExceptGroupStmt getAGroupHandler() { result = this.getAHandler() }
 
   override Stmt getLastStatement() {
     result = this.getFinalbody().getLastItem().getLastStatement()

python/ql/lib/semmle/python/dataflow/new/internal/FlowSummaryImpl.qll

@@ -260,6 +260,12 @@ module Public {
      *  Holds if the neutral is auto generated.
      */
     predicate isAutoGenerated() { neutralElement(this, true) }
+
+    /**
+     * Holds if the neutral has the given provenance where `true` is
+     * `generated` and `false` is `manual`.
+     */
+    predicate hasProvenance(boolean generated) { neutralElement(this, generated) }
   }
 }
 

python/ql/lib/semmle/python/essa/Definitions.qll

@@ -69,6 +69,8 @@ abstract class SsaSourceVariable extends @py_variable {
     or
     SsaSource::exception_capture(this, def)
     or
+    SsaSource::exception_group_capture(this, def)
+    or
     SsaSource::with_definition(this, def)
     or
     SsaSource::pattern_capture_definition(this, def)

python/ql/lib/semmle/python/essa/Essa.qll

@@ -511,12 +511,16 @@ class AssignmentDefinition extends EssaNodeDefinition {
   override string getAPrimaryQlClass() { result = "AssignmentDefinition" }
 }
 
-/** A capture of a raised exception `except ExceptionType ex:` */
+/** A capture of a raised exception `except ExceptionType as ex:` */
 class ExceptionCapture extends EssaNodeDefinition {
   ExceptionCapture() {
     SsaSource::exception_capture(this.getSourceVariable(), this.getDefiningNode())
   }
 
+  /**
+   * Gets the type handled by this exception handler
+   * `ExceptionType` in `except ExceptionType as ex:`.
+   */
   ControlFlowNode getType() {
     exists(ExceptFlowNode ex |
       ex.getName() = this.getDefiningNode() and
@@ -529,6 +533,28 @@ class ExceptionCapture extends EssaNodeDefinition {
   override string getAPrimaryQlClass() { result = "ExceptionCapture" }
 }
 
+/** A capture of a raised exception group `except* ExceptionType as ex:` */
+class ExceptionGroupCapture extends EssaNodeDefinition {
+  ExceptionGroupCapture() {
+    SsaSource::exception_group_capture(this.getSourceVariable(), this.getDefiningNode())
+  }
+
+  /**
+   * Gets the type handled by this exception handler
+   * `ExceptionType` in `except* ExceptionType as ex:`.
+   */
+  ControlFlowNode getType() {
+    exists(ExceptGroupFlowNode ex |
+      ex.getName() = this.getDefiningNode() and
+      result = ex.getType()
+    )
+  }
+
+  override string getRepresentation() { result = "except* " + this.getSourceVariable().getName() }
+
+  override string getAPrimaryQlClass() { result = "ExceptionGroupCapture" }
+}
+
 /** An assignment to a variable as part of a multiple assignment `..., v, ... = val` */
 class MultiAssignmentDefinition extends EssaNodeDefinition {
   MultiAssignmentDefinition() {

python/ql/lib/semmle/python/essa/SsaDefinitions.qll

@@ -30,6 +30,13 @@ module SsaSource {
     exists(ExceptFlowNode ex | ex.getName() = defn)
   }
 
+  /** Holds if `v` is defined by assignment of the captured exception group. */
+  cached
+  predicate exception_group_capture(Variable v, NameNode defn) {
+    defn.defines(v) and
+    exists(ExceptGroupFlowNode ex | ex.getName() = defn)
+  }
+
   /** Holds if `v` is defined by a with statement. */
   cached
   predicate with_definition(Variable v, ControlFlowNode defn) {

python/ql/lib/semmle/python/types/Exceptions.qll

@@ -367,6 +367,10 @@ predicate scope_raises_unknown(Scope s) {
 class ExceptFlowNode extends ControlFlowNode {
   ExceptFlowNode() { this.getNode() instanceof ExceptStmt }
 
+  /**
+   * Gets the type handled by this exception handler.
+   * `ExceptionType` in `except ExceptionType as e:`
+   */
   ControlFlowNode getType() {
     exists(ExceptStmt ex |
       this.getBasicBlock().dominates(result.getBasicBlock()) and
@@ -375,6 +379,10 @@ class ExceptFlowNode extends ControlFlowNode {
     )
   }
 
+  /**
+   * Gets the name assigned to the handled exception, if any.
+   * `e` in `except ExceptionType as e:`
+   */
   ControlFlowNode getName() {
     exists(ExceptStmt ex |
       this.getBasicBlock().dominates(result.getBasicBlock()) and
@@ -439,6 +447,29 @@ class ExceptFlowNode extends ControlFlowNode {
   }
 }
 
+/** The ControlFlowNode for an 'except*' statement. */
+class ExceptGroupFlowNode extends ControlFlowNode {
+  ExceptGroupFlowNode() { this.getNode() instanceof ExceptGroupStmt }
+
+  /**
+   * Gets the type handled by this exception handler.
+   * `ExceptionType` in `except* ExceptionType as e:`
+   */
+  ControlFlowNode getType() {
+    this.getBasicBlock().dominates(result.getBasicBlock()) and
+    result = this.getNode().(ExceptGroupStmt).getType().getAFlowNode()
+  }
+
+  /**
+   * Gets the name assigned to the handled exception, if any.
+   * `e` in `except* ExceptionType as e:`
+   */
+  ControlFlowNode getName() {
+    this.getBasicBlock().dominates(result.getBasicBlock()) and
+    result = this.getNode().(ExceptGroupStmt).getName().getAFlowNode()
+  }
+}
+
 private ControlFlowNode element_from_tuple_objectapi(Object tuple) {
   exists(Tuple t | t = tuple.getOrigin() and result = t.getAnElt().getAFlowNode())
 }

python/ql/lib/semmlecode.python.dbscheme

@@ -273,6 +273,11 @@ py_extracted_version(int module : @py_Module ref,
 /* <Field> Ellipsis.location = 0, location */
 /* <Field> Ellipsis.parenthesised = 1, bool */
 
+/* <Field> ExceptGroupStmt.location = 0, location */
+/* <Field> ExceptGroupStmt.type = 1, expr */
+/* <Field> ExceptGroupStmt.name = 2, expr */
+/* <Field> ExceptGroupStmt.body = 3, stmt_list */
+
 /* <Field> ExceptStmt.location = 0, location */
 /* <Field> ExceptStmt.type = 1, expr */
 /* <Field> ExceptStmt.name = 2, expr */
@@ -863,25 +868,26 @@ case @py_stmt.kind of
 |   4 = @py_Continue
 |   5 = @py_Delete
 |   6 = @py_ExceptStmt
-|   7 = @py_Exec
-|   8 = @py_Expr_stmt
-|   9 = @py_For
-|   10 = @py_Global
-|   11 = @py_If
-|   12 = @py_Import
-|   13 = @py_ImportStar
-|   14 = @py_MatchStmt
-|   15 = @py_Case
-|   16 = @py_Nonlocal
-|   17 = @py_Pass
-|   18 = @py_Print
-|   19 = @py_Raise
-|   20 = @py_Return
-|   21 = @py_Try
-|   22 = @py_While
-|   23 = @py_With
-|   24 = @py_TemplateWrite
-|   25 = @py_AnnAssign;
+|   7 = @py_ExceptGroupStmt
+|   8 = @py_Exec
+|   9 = @py_Expr_stmt
+|   10 = @py_For
+|   11 = @py_Global
+|   12 = @py_If
+|   13 = @py_Import
+|   14 = @py_ImportStar
+|   15 = @py_MatchStmt
+|   16 = @py_Case
+|   17 = @py_Nonlocal
+|   18 = @py_Pass
+|   19 = @py_Print
+|   20 = @py_Raise
+|   21 = @py_Return
+|   22 = @py_Try
+|   23 = @py_While
+|   24 = @py_With
+|   25 = @py_TemplateWrite
+|   26 = @py_AnnAssign;
 
 case @py_unaryop.kind of
     0 = @py_Invert
@@ -907,7 +913,7 @@ case @py_unaryop.kind of
 
 @py_expr_or_stmt = @py_expr | @py_stmt;
 
-@py_expr_parent = @py_AnnAssign | @py_Assert | @py_Assign | @py_AssignExpr | @py_Attribute | @py_AugAssign | @py_Await | @py_BinaryExpr | @py_Call | @py_Case | @py_Compare | @py_DictComp | @py_DictUnpacking | @py_ExceptStmt | @py_Exec | @py_Expr_stmt | @py_Filter | @py_For | @py_FormattedValue | @py_Function | @py_FunctionExpr | @py_GeneratorExp | @py_Guard | @py_If | @py_IfExp | @py_ImportMember | @py_ImportStar | @py_KeyValuePair | @py_ListComp | @py_MatchAsPattern | @py_MatchCapturePattern | @py_MatchClassPattern | @py_MatchKeywordPattern | @py_MatchLiteralPattern | @py_MatchStmt | @py_MatchValuePattern | @py_Print | @py_Raise | @py_Repr | @py_Return | @py_SetComp | @py_Slice | @py_Starred | @py_Subscript | @py_TemplateDottedNotation | @py_TemplateWrite | @py_UnaryExpr | @py_While | @py_With | @py_Yield | @py_YieldFrom | @py_alias | @py_arguments | @py_comprehension | @py_expr_list | @py_keyword | @py_parameter_list;
+@py_expr_parent = @py_AnnAssign | @py_Assert | @py_Assign | @py_AssignExpr | @py_Attribute | @py_AugAssign | @py_Await | @py_BinaryExpr | @py_Call | @py_Case | @py_Compare | @py_DictComp | @py_DictUnpacking | @py_ExceptGroupStmt | @py_ExceptStmt | @py_Exec | @py_Expr_stmt | @py_Filter | @py_For | @py_FormattedValue | @py_Function | @py_FunctionExpr | @py_GeneratorExp | @py_Guard | @py_If | @py_IfExp | @py_ImportMember | @py_ImportStar | @py_KeyValuePair | @py_ListComp | @py_MatchAsPattern | @py_MatchCapturePattern | @py_MatchClassPattern | @py_MatchKeywordPattern | @py_MatchLiteralPattern | @py_MatchStmt | @py_MatchValuePattern | @py_Print | @py_Raise | @py_Repr | @py_Return | @py_SetComp | @py_Slice | @py_Starred | @py_Subscript | @py_TemplateDottedNotation | @py_TemplateWrite | @py_UnaryExpr | @py_While | @py_With | @py_Yield | @py_YieldFrom | @py_alias | @py_arguments | @py_comprehension | @py_expr_list | @py_keyword | @py_parameter_list;
 
 @py_location_parent = @py_DictUnpacking | @py_KeyValuePair | @py_StringPart | @py_comprehension | @py_expr | @py_keyword | @py_pattern | @py_stmt;
 
@@ -919,7 +925,7 @@ case @py_unaryop.kind of
 
 @py_scope = @py_Class | @py_Function | @py_Module;
 
-@py_stmt_list_parent = @py_Case | @py_Class | @py_ExceptStmt | @py_For | @py_Function | @py_If | @py_MatchStmt | @py_Module | @py_Try | @py_While | @py_With;
+@py_stmt_list_parent = @py_Case | @py_Class | @py_ExceptGroupStmt | @py_ExceptStmt | @py_For | @py_Function | @py_If | @py_MatchStmt | @py_Module | @py_Try | @py_While | @py_With;
 
 @py_str_list_parent = @py_Global | @py_Nonlocal;
 

python/ql/lib/semmlecode.python.dbscheme.stats

@@ -469,6 +469,10 @@
 <v>5610</v>
 </e>
 <e>
+<k>@py_ExceptGroupStmt</k>
+<v>1000</v>
+</e>
+<e>
 <k>@py_Expr_stmt</k>
 <v>76750</v>
 </e>

python/ql/lib/upgrades/503c0516fba2e5da9570f00eb34ef43025ecb8fb/py_stmts.ql

@@ -0,0 +1,30 @@
+// First we need to wrap some database types
+class Stmt_ extends @py_stmt {
+  string toString() { result = "Stmt" }
+}
+
+class StmtList_ extends @py_stmt_list {
+  string toString() { result = "StmtList" }
+}
+
+/**
+ * New kinds have been inserted such that
+ * `@py_Exec` which used to have index 7 now has index 8.
+ * Entries with lower indices are unchanged.
+ */
+bindingset[old_index]
+int new_index(int old_index) {
+  if old_index < 7 then result = old_index else result = (8 - 7) + old_index
+}
+
+// The schema for py_stmts is:
+//
+// py_stmts(unique int id : @py_stmt,
+//   int kind: int ref,
+//   int parent : @py_stmt_list ref,
+//   int idx : int ref);
+from Stmt_ expr, int old_kind, StmtList_ parent, int idx, int new_kind
+where
+  py_stmts(expr, old_kind, parent, idx) and
+  new_kind = new_index(old_kind)
+select expr, new_kind, parent, idx

python/ql/lib/upgrades/503c0516fba2e5da9570f00eb34ef43025ecb8fb/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add support for `except*`
+compatibility: backwards
+py_stmts.rel: run py_stmts.qlo