Merge branch 'main' into calumgrant/remove-lgtm

python/config/suites/lgtm/python-queries-lgtm

@@ -1,6 +1,6 @@
 + semmlecode-python-queries/analysis/Definitions.ql
     @_namespace com.lgtm/python-queries
-+ semmlecode-python-queries/analysis/AlertSuppression.ql
++ semmlecode-python-queries/AlertSuppression.ql
     @_namespace com.lgtm/python-queries
 + semmlecode-python-queries/Filters/ClassifyFiles.ql
     @_namespace com.lgtm/python-queries

python/downgrades/47e552c4357a04c5735355fad818630daee4a5ac/py_stmts.ql

@@ -0,0 +1,31 @@
+// First we need to wrap some database types
+class Stmt_ extends @py_stmt {
+  string toString() { result = "Stmt" }
+}
+
+class StmtList_ extends @py_stmt_list {
+  string toString() { result = "StmtList" }
+}
+
+/**
+ * New kinds have been inserted such that
+ * `@py_Exec` which used to have index 7 now has index 8.
+ * Entries with lower indices are unchanged.
+ */
+bindingset[new_index]
+int old_index(int new_index) {
+  not new_index = 7 and
+  if new_index < 7 then result = new_index else result + (8 - 7) = new_index
+}
+
+// The schema for py_stmts is:
+//
+// py_stmts(unique int id : @py_stmt,
+//   int kind: int ref,
+//   int parent : @py_stmt_list ref,
+//   int idx : int ref);
+from Stmt_ expr, int new_kind, StmtList_ parent, int idx, int old_kind
+where
+  py_stmts(expr, new_kind, parent, idx) and
+  old_kind = old_index(new_kind)
+select expr, old_kind, parent, idx

python/downgrades/47e552c4357a04c5735355fad818630daee4a5ac/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add support for `except*`
+compatibility: backwards
+py_stmts.rel: run py_stmts.qlo

python/ql/lib/change-notes/2022-11-14-grouped-exceptions.md

@@ -0,0 +1,5 @@
+---
+category: fix
+---
+* `except*` is now supported.
+* The result of `Try.getAHandler` and `Try.getHandler(<index>)` is no longer of type `ExceptStmt`, as handlers may also be `ExceptGroupStmt`s (After Python 3.11 introduced PEP 654). Instead, it is of the new type `ExceptionHandler` of which `ExceptStmt` and `ExceptGroupStmt` are subtypes. To support selecting only one type of handler, `Try.getANormalHandler` and `Try.getAGroupHandler` have been added. Existing uses of `Try.getAHandler` for which it is important to select only normal handlers, will need to be updated to `Try.getANormalHandler`.

python/ql/lib/semmle/python/AstGenerated.qll

@@ -389,6 +389,26 @@ class Eq_ extends @py_Eq, Cmpop {
   override string toString() { result = "Eq" }
 }
 
+/** INTERNAL: See the class `ExceptGroupStmt` for further information. */
+class ExceptGroupStmt_ extends @py_ExceptGroupStmt, Stmt {
+  /** Gets the type of this except group block. */
+  Expr getType() { py_exprs(result, _, this, 1) }
+
+  /** Gets the name of this except group block. */
+  Expr getName() { py_exprs(result, _, this, 2) }
+
+  /** Gets the body of this except group block. */
+  StmtList getBody() { py_stmt_lists(result, this, 3) }
+
+  /** Gets the nth statement of this except group block. */
+  Stmt getStmt(int index) { result = this.getBody().getItem(index) }
+
+  /** Gets a statement of this except group block. */
+  Stmt getAStmt() { result = this.getBody().getAnItem() }
+
+  override string toString() { result = "ExceptGroupStmt" }
+}
+
 /** INTERNAL: See the class `ExceptStmt` for further information. */
 class ExceptStmt_ extends @py_ExceptStmt, Stmt {
   /** Gets the type of this except block. */

python/ql/lib/semmle/python/Stmts.qll

@@ -143,12 +143,30 @@ class Exec extends Exec_ {
   override Stmt getASubStatement() { none() }
 }
 
-/** An except statement (part of a `try` statement), such as `except IOError as err:` */
-class ExceptStmt extends ExceptStmt_ {
-  /* syntax: except Expr [ as Expr ]: */
+/**
+ * An exception handler such as an `except` or an `except*` statement
+ * in a `try` statement.
+ */
+class ExceptionHandler extends Stmt {
+  ExceptionHandler() {
+    this instanceof ExceptStmt_
+    or
+    this instanceof ExceptGroupStmt_
+  }
+
   /** Gets the immediately enclosing try statement */
   Try getTry() { result.getAHandler() = this }
 
+  /** Gets the name of this except group block. */
+  abstract Expr getName();
+
+  /** Gets the type of this except group block. */
+  abstract Expr getType();
+}
+
+/** An except group statement (part of a `try` statement), such as `except* IOError as err:` */
+class ExceptGroupStmt extends ExceptGroupStmt_, ExceptionHandler {
+  /* syntax: except Expr [ as Expr ]: */
   override Expr getASubExpression() {
     result = this.getName()
     or
@@ -159,10 +177,34 @@ class ExceptStmt extends ExceptStmt_ {
 
   override Stmt getLastStatement() { result = this.getBody().getLastItem().getLastStatement() }
 
+  override Expr getName() { result = ExceptGroupStmt_.super.getName() }
+
   override Expr getType() {
-    result = super.getType() and not result instanceof Tuple
+    result = ExceptGroupStmt_.super.getType() and not result instanceof Tuple
     or
-    result = super.getType().(Tuple).getAnElt()
+    result = ExceptGroupStmt_.super.getType().(Tuple).getAnElt()
+  }
+}
+
+/** An except statement (part of a `try` statement), such as `except IOError as err:` */
+class ExceptStmt extends ExceptStmt_, ExceptionHandler {
+  /* syntax: except Expr [ as Expr ]: */
+  override Expr getASubExpression() {
+    result = this.getName()
+    or
+    result = this.getType()
+  }
+
+  override Stmt getASubStatement() { result = this.getAStmt() }
+
+  override Stmt getLastStatement() { result = this.getBody().getLastItem().getLastStatement() }
+
+  override Expr getName() { result = ExceptStmt_.super.getName() }
+
+  override Expr getType() {
+    result = ExceptStmt_.super.getType() and not result instanceof Tuple
+    or
+    result = ExceptStmt_.super.getType().(Tuple).getAnElt()
   }
 }
 
@@ -364,10 +406,15 @@ class Try extends Try_ {
     result = this.getAnOrelse()
   }
 
-  override ExceptStmt getHandler(int i) { result = Try_.super.getHandler(i) }
+  override ExceptionHandler getHandler(int i) { result = Try_.super.getHandler(i) }
 
-  /** Gets an exception handler of this try statement. */
-  override ExceptStmt getAHandler() { result = Try_.super.getAHandler() }
+  override ExceptionHandler getAHandler() { result = Try_.super.getAHandler() }
+
+  /** Gets a normal exception handler, `except`, of this try statement. */
+  ExceptStmt getANormalHandler() { result = this.getAHandler() }
+
+  /** Gets a group exception handler, `except*`, of this try statement. */
+  ExceptGroupStmt getAGroupHandler() { result = this.getAHandler() }
 
   override Stmt getLastStatement() {
     result = this.getFinalbody().getLastItem().getLastStatement()

python/ql/lib/semmle/python/dataflow/new/internal/FlowSummaryImpl.qll

@@ -260,6 +260,12 @@ module Public {
      *  Holds if the neutral is auto generated.
      */
     predicate isAutoGenerated() { neutralElement(this, true) }
+
+    /**
+     * Holds if the neutral has the given provenance where `true` is
+     * `generated` and `false` is `manual`.
+     */
+    predicate hasProvenance(boolean generated) { neutralElement(this, generated) }
   }
 }
 

python/ql/lib/semmle/python/essa/Definitions.qll

@@ -69,6 +69,8 @@ abstract class SsaSourceVariable extends @py_variable {
     or
     SsaSource::exception_capture(this, def)
     or
+    SsaSource::exception_group_capture(this, def)
+    or
     SsaSource::with_definition(this, def)
     or
     SsaSource::pattern_capture_definition(this, def)

python/ql/lib/semmle/python/essa/Essa.qll

@@ -511,12 +511,16 @@ class AssignmentDefinition extends EssaNodeDefinition {
   override string getAPrimaryQlClass() { result = "AssignmentDefinition" }
 }
 
-/** A capture of a raised exception `except ExceptionType ex:` */
+/** A capture of a raised exception `except ExceptionType as ex:` */
 class ExceptionCapture extends EssaNodeDefinition {
   ExceptionCapture() {
     SsaSource::exception_capture(this.getSourceVariable(), this.getDefiningNode())
   }
 
+  /**
+   * Gets the type handled by this exception handler
+   * `ExceptionType` in `except ExceptionType as ex:`.
+   */
   ControlFlowNode getType() {
     exists(ExceptFlowNode ex |
       ex.getName() = this.getDefiningNode() and
@@ -529,6 +533,28 @@ class ExceptionCapture extends EssaNodeDefinition {
   override string getAPrimaryQlClass() { result = "ExceptionCapture" }
 }
 
+/** A capture of a raised exception group `except* ExceptionType as ex:` */
+class ExceptionGroupCapture extends EssaNodeDefinition {
+  ExceptionGroupCapture() {
+    SsaSource::exception_group_capture(this.getSourceVariable(), this.getDefiningNode())
+  }
+
+  /**
+   * Gets the type handled by this exception handler
+   * `ExceptionType` in `except* ExceptionType as ex:`.
+   */
+  ControlFlowNode getType() {
+    exists(ExceptGroupFlowNode ex |
+      ex.getName() = this.getDefiningNode() and
+      result = ex.getType()
+    )
+  }
+
+  override string getRepresentation() { result = "except* " + this.getSourceVariable().getName() }
+
+  override string getAPrimaryQlClass() { result = "ExceptionGroupCapture" }
+}
+
 /** An assignment to a variable as part of a multiple assignment `..., v, ... = val` */
 class MultiAssignmentDefinition extends EssaNodeDefinition {
   MultiAssignmentDefinition() {

python/ql/lib/semmle/python/essa/SsaDefinitions.qll

@@ -30,6 +30,13 @@ module SsaSource {
     exists(ExceptFlowNode ex | ex.getName() = defn)
   }
 
+  /** Holds if `v` is defined by assignment of the captured exception group. */
+  cached
+  predicate exception_group_capture(Variable v, NameNode defn) {
+    defn.defines(v) and
+    exists(ExceptGroupFlowNode ex | ex.getName() = defn)
+  }
+
   /** Holds if `v` is defined by a with statement. */
   cached
   predicate with_definition(Variable v, ControlFlowNode defn) {

python/ql/lib/semmle/python/types/Exceptions.qll

@@ -367,6 +367,10 @@ predicate scope_raises_unknown(Scope s) {
 class ExceptFlowNode extends ControlFlowNode {
   ExceptFlowNode() { this.getNode() instanceof ExceptStmt }
 
+  /**
+   * Gets the type handled by this exception handler.
+   * `ExceptionType` in `except ExceptionType as e:`
+   */
   ControlFlowNode getType() {
     exists(ExceptStmt ex |
       this.getBasicBlock().dominates(result.getBasicBlock()) and
@@ -375,6 +379,10 @@ class ExceptFlowNode extends ControlFlowNode {
     )
   }
 
+  /**
+   * Gets the name assigned to the handled exception, if any.
+   * `e` in `except ExceptionType as e:`
+   */
   ControlFlowNode getName() {
     exists(ExceptStmt ex |
       this.getBasicBlock().dominates(result.getBasicBlock()) and
@@ -439,6 +447,29 @@ class ExceptFlowNode extends ControlFlowNode {
   }
 }
 
+/** The ControlFlowNode for an 'except*' statement. */
+class ExceptGroupFlowNode extends ControlFlowNode {
+  ExceptGroupFlowNode() { this.getNode() instanceof ExceptGroupStmt }
+
+  /**
+   * Gets the type handled by this exception handler.
+   * `ExceptionType` in `except* ExceptionType as e:`
+   */
+  ControlFlowNode getType() {
+    this.getBasicBlock().dominates(result.getBasicBlock()) and
+    result = this.getNode().(ExceptGroupStmt).getType().getAFlowNode()
+  }
+
+  /**
+   * Gets the name assigned to the handled exception, if any.
+   * `e` in `except* ExceptionType as e:`
+   */
+  ControlFlowNode getName() {
+    this.getBasicBlock().dominates(result.getBasicBlock()) and
+    result = this.getNode().(ExceptGroupStmt).getName().getAFlowNode()
+  }
+}
+
 private ControlFlowNode element_from_tuple_objectapi(Object tuple) {
   exists(Tuple t | t = tuple.getOrigin() and result = t.getAnElt().getAFlowNode())
 }

python/ql/lib/semmlecode.python.dbscheme

@@ -273,6 +273,11 @@ py_extracted_version(int module : @py_Module ref,
 /* <Field> Ellipsis.location = 0, location */
 /* <Field> Ellipsis.parenthesised = 1, bool */
 
+/* <Field> ExceptGroupStmt.location = 0, location */
+/* <Field> ExceptGroupStmt.type = 1, expr */
+/* <Field> ExceptGroupStmt.name = 2, expr */
+/* <Field> ExceptGroupStmt.body = 3, stmt_list */
+
 /* <Field> ExceptStmt.location = 0, location */
 /* <Field> ExceptStmt.type = 1, expr */
 /* <Field> ExceptStmt.name = 2, expr */
@@ -863,25 +868,26 @@ case @py_stmt.kind of
 |   4 = @py_Continue
 |   5 = @py_Delete
 |   6 = @py_ExceptStmt
-|   7 = @py_Exec
-|   8 = @py_Expr_stmt
-|   9 = @py_For
-|   10 = @py_Global
-|   11 = @py_If
-|   12 = @py_Import
-|   13 = @py_ImportStar
-|   14 = @py_MatchStmt
-|   15 = @py_Case
-|   16 = @py_Nonlocal
-|   17 = @py_Pass
-|   18 = @py_Print
-|   19 = @py_Raise
-|   20 = @py_Return
-|   21 = @py_Try
-|   22 = @py_While
-|   23 = @py_With
-|   24 = @py_TemplateWrite
-|   25 = @py_AnnAssign;
+|   7 = @py_ExceptGroupStmt
+|   8 = @py_Exec
+|   9 = @py_Expr_stmt
+|   10 = @py_For
+|   11 = @py_Global
+|   12 = @py_If
+|   13 = @py_Import
+|   14 = @py_ImportStar
+|   15 = @py_MatchStmt
+|   16 = @py_Case
+|   17 = @py_Nonlocal
+|   18 = @py_Pass
+|   19 = @py_Print
+|   20 = @py_Raise
+|   21 = @py_Return
+|   22 = @py_Try
+|   23 = @py_While
+|   24 = @py_With
+|   25 = @py_TemplateWrite
+|   26 = @py_AnnAssign;
 
 case @py_unaryop.kind of
     0 = @py_Invert
@@ -907,7 +913,7 @@ case @py_unaryop.kind of
 
 @py_expr_or_stmt = @py_expr | @py_stmt;
 
-@py_expr_parent = @py_AnnAssign | @py_Assert | @py_Assign | @py_AssignExpr | @py_Attribute | @py_AugAssign | @py_Await | @py_BinaryExpr | @py_Call | @py_Case | @py_Compare | @py_DictComp | @py_DictUnpacking | @py_ExceptStmt | @py_Exec | @py_Expr_stmt | @py_Filter | @py_For | @py_FormattedValue | @py_Function | @py_FunctionExpr | @py_GeneratorExp | @py_Guard | @py_If | @py_IfExp | @py_ImportMember | @py_ImportStar | @py_KeyValuePair | @py_ListComp | @py_MatchAsPattern | @py_MatchCapturePattern | @py_MatchClassPattern | @py_MatchKeywordPattern | @py_MatchLiteralPattern | @py_MatchStmt | @py_MatchValuePattern | @py_Print | @py_Raise | @py_Repr | @py_Return | @py_SetComp | @py_Slice | @py_Starred | @py_Subscript | @py_TemplateDottedNotation | @py_TemplateWrite | @py_UnaryExpr | @py_While | @py_With | @py_Yield | @py_YieldFrom | @py_alias | @py_arguments | @py_comprehension | @py_expr_list | @py_keyword | @py_parameter_list;
+@py_expr_parent = @py_AnnAssign | @py_Assert | @py_Assign | @py_AssignExpr | @py_Attribute | @py_AugAssign | @py_Await | @py_BinaryExpr | @py_Call | @py_Case | @py_Compare | @py_DictComp | @py_DictUnpacking | @py_ExceptGroupStmt | @py_ExceptStmt | @py_Exec | @py_Expr_stmt | @py_Filter | @py_For | @py_FormattedValue | @py_Function | @py_FunctionExpr | @py_GeneratorExp | @py_Guard | @py_If | @py_IfExp | @py_ImportMember | @py_ImportStar | @py_KeyValuePair | @py_ListComp | @py_MatchAsPattern | @py_MatchCapturePattern | @py_MatchClassPattern | @py_MatchKeywordPattern | @py_MatchLiteralPattern | @py_MatchStmt | @py_MatchValuePattern | @py_Print | @py_Raise | @py_Repr | @py_Return | @py_SetComp | @py_Slice | @py_Starred | @py_Subscript | @py_TemplateDottedNotation | @py_TemplateWrite | @py_UnaryExpr | @py_While | @py_With | @py_Yield | @py_YieldFrom | @py_alias | @py_arguments | @py_comprehension | @py_expr_list | @py_keyword | @py_parameter_list;
 
 @py_location_parent = @py_DictUnpacking | @py_KeyValuePair | @py_StringPart | @py_comprehension | @py_expr | @py_keyword | @py_pattern | @py_stmt;
 
@@ -919,7 +925,7 @@ case @py_unaryop.kind of
 
 @py_scope = @py_Class | @py_Function | @py_Module;
 
-@py_stmt_list_parent = @py_Case | @py_Class | @py_ExceptStmt | @py_For | @py_Function | @py_If | @py_MatchStmt | @py_Module | @py_Try | @py_While | @py_With;
+@py_stmt_list_parent = @py_Case | @py_Class | @py_ExceptGroupStmt | @py_ExceptStmt | @py_For | @py_Function | @py_If | @py_MatchStmt | @py_Module | @py_Try | @py_While | @py_With;
 
 @py_str_list_parent = @py_Global | @py_Nonlocal;
 

python/ql/lib/semmlecode.python.dbscheme.stats

@@ -469,6 +469,10 @@
 <v>5610</v>
 </e>
 <e>
+<k>@py_ExceptGroupStmt</k>
+<v>1000</v>
+</e>
+<e>
 <k>@py_Expr_stmt</k>
 <v>76750</v>
 </e>

python/ql/lib/upgrades/503c0516fba2e5da9570f00eb34ef43025ecb8fb/py_stmts.ql

@@ -0,0 +1,30 @@
+// First we need to wrap some database types
+class Stmt_ extends @py_stmt {
+  string toString() { result = "Stmt" }
+}
+
+class StmtList_ extends @py_stmt_list {
+  string toString() { result = "StmtList" }
+}
+
+/**
+ * New kinds have been inserted such that
+ * `@py_Exec` which used to have index 7 now has index 8.
+ * Entries with lower indices are unchanged.
+ */
+bindingset[old_index]
+int new_index(int old_index) {
+  if old_index < 7 then result = old_index else result = (8 - 7) + old_index
+}
+
+// The schema for py_stmts is:
+//
+// py_stmts(unique int id : @py_stmt,
+//   int kind: int ref,
+//   int parent : @py_stmt_list ref,
+//   int idx : int ref);
+from Stmt_ expr, int old_kind, StmtList_ parent, int idx, int new_kind
+where
+  py_stmts(expr, old_kind, parent, idx) and
+  new_kind = new_index(old_kind)
+select expr, new_kind, parent, idx

python/ql/lib/upgrades/503c0516fba2e5da9570f00eb34ef43025ecb8fb/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add support for `except*`
+compatibility: backwards
+py_stmts.rel: run py_stmts.qlo

python/ql/src/AlertSuppression.ql

@@ -0,0 +1,51 @@
+/**
+ * @name Alert suppression
+ * @description Generates information about alert suppressions.
+ * @kind alert-suppression
+ * @id py/alert-suppression
+ */
+
+private import codeql.util.suppression.AlertSuppression as AS
+private import semmle.python.Comment as P
+
+class AstNode instanceof P::AstNode {
+  predicate hasLocationInfo(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  ) {
+    super.getLocation().hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+  }
+
+  string toString() { result = super.toString() }
+}
+
+class SingleLineComment instanceof P::Comment {
+  predicate hasLocationInfo(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  ) {
+    super.getLocation().hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+  }
+
+  string getText() { result = super.getContents() }
+
+  string toString() { result = super.toString() }
+}
+
+import AS::Make<AstNode, SingleLineComment>
+
+/**
+ * A noqa suppression comment. Both pylint and pyflakes respect this, so lgtm ought to too.
+ */
+class NoqaSuppressionComment extends SuppressionComment instanceof SingleLineComment {
+  NoqaSuppressionComment() {
+    SingleLineComment.super.getText().regexpMatch("(?i)\\s*noqa\\s*([^:].*)?")
+  }
+
+  override string getAnnotation() { result = "lgtm" }
+
+  override predicate covers(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  ) {
+    this.hasLocationInfo(filepath, startline, _, endline, endcolumn) and
+    startcolumn = 1
+  }
+}

python/ql/src/analysis/AlertSuppression.ql

@@ -1,105 +0,0 @@
-/**
- * @name Alert suppression
- * @description Generates information about alert suppressions.
- * @kind alert-suppression
- * @id py/alert-suppression
- */
-
-import python
-
-/**
- * An alert suppression comment.
- */
-abstract class SuppressionComment extends Comment {
-  /** Gets the scope of this suppression. */
-  abstract SuppressionScope getScope();
-
-  /** Gets the suppression annotation in this comment. */
-  abstract string getAnnotation();
-
-  /**
-   * Holds if this comment applies to the range from column `startcolumn` of line `startline`
-   * to column `endcolumn` of line `endline` in file `filepath`.
-   */
-  abstract predicate covers(
-    string filepath, int startline, int startcolumn, int endline, int endcolumn
-  );
-}
-
-/**
- * An alert comment that applies to a single line
- */
-abstract class LineSuppressionComment extends SuppressionComment {
-  LineSuppressionComment() {
-    exists(string filepath, int l |
-      this.getLocation().hasLocationInfo(filepath, l, _, _, _) and
-      any(AstNode a).getLocation().hasLocationInfo(filepath, l, _, _, _)
-    )
-  }
-
-  /** Gets the scope of this suppression. */
-  override SuppressionScope getScope() { result = this }
-
-  override predicate covers(
-    string filepath, int startline, int startcolumn, int endline, int endcolumn
-  ) {
-    this.getLocation().hasLocationInfo(filepath, startline, _, endline, endcolumn) and
-    startcolumn = 1
-  }
-}
-
-/**
- * An lgtm suppression comment.
- */
-class LgtmSuppressionComment extends LineSuppressionComment {
-  string annotation;
-
-  LgtmSuppressionComment() {
-    exists(string all | all = this.getContents() |
-      // match `lgtm[...]` anywhere in the comment
-      annotation = all.regexpFind("(?i)\\blgtm\\s*\\[[^\\]]*\\]", _, _)
-      or
-      // match `lgtm` at the start of the comment and after semicolon
-      annotation = all.regexpFind("(?i)(?<=^|;)\\s*lgtm(?!\\B|\\s*\\[)", _, _).trim()
-    )
-  }
-
-  /** Gets the suppression annotation in this comment. */
-  override string getAnnotation() { result = annotation }
-}
-
-/**
- * A noqa suppression comment. Both pylint and pyflakes respect this, so lgtm ought to too.
- */
-class NoqaSuppressionComment extends LineSuppressionComment {
-  NoqaSuppressionComment() { this.getContents().toLowerCase().regexpMatch("\\s*noqa\\s*([^:].*)?") }
-
-  override string getAnnotation() { result = "lgtm" }
-}
-
-/**
- * The scope of an alert suppression comment.
- */
-class SuppressionScope extends @py_comment instanceof SuppressionComment {
-  /**
-   * Holds if this element is at the specified location.
-   * The location spans column `startcolumn` of line `startline` to
-   * column `endcolumn` of line `endline` in file `filepath`.
-   * For more information, see
-   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
-   */
-  predicate hasLocationInfo(
-    string filepath, int startline, int startcolumn, int endline, int endcolumn
-  ) {
-    super.covers(filepath, startline, startcolumn, endline, endcolumn)
-  }
-
-  /** Gets a textual representation of this element. */
-  string toString() { result = "suppression range" }
-}
-
-from SuppressionComment c
-select c, // suppression comment
-  c.getContents(), // text of suppression comment (excluding delimiters)
-  c.getAnnotation(), // text of suppression annotation
-  c.getScope() // scope of suppression

python/ql/src/change-notes/2022-12-19-alert-suppressions.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The `analysis/AlertSuppression.ql` query has moved to the root folder. Users that refer to this query by path should update their configurations. The query has been updated to support the new `# codeql[query-id]` supression comments. These comments can be used to suppress an alert and must be placed on a blank line before the alert. In addition the legacy `# lgtm` and `# lgtm[query-id]` comments can now also be place on the line before an alert.

python/ql/src/qlpack.yml

@@ -6,6 +6,7 @@ groups:
 dependencies:
   codeql/python-all: ${workspace}
   codeql/suite-helpers: ${workspace}
+  codeql/util: ${workspace}
 suites: codeql-suites
 extractor: python
 defaultSuiteFile: codeql-suites/python-code-scanning.qls

python/ql/test/experimental/dataflow/coverage/classes.py

@@ -1521,7 +1521,7 @@ class With_await:
     def __await__(self):
         SINK1(self)
         OK()  # Call not found
-        return (yield from asyncio.coroutine(lambda: "")())
+        return (yield from [])
 
 
 async def atest_await():

python/ql/test/experimental/dataflow/exceptions/NormalDataflowTest.expected

@@ -0,0 +1,2 @@
+missingAnnotationOnSink
+failures

python/ql/test/experimental/dataflow/exceptions/NormalDataflowTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.dataflow.TestUtil.NormalDataflowTest

python/ql/test/experimental/dataflow/exceptions/dataflow-consistency.expected

@@ -0,0 +1,24 @@
+uniqueEnclosingCallable
+uniqueType
+uniqueNodeLocation
+missingLocation
+uniqueNodeToString
+missingToString
+parameterCallable
+localFlowIsLocal
+readStepIsLocal
+storeStepIsLocal
+compatibleTypesReflexive
+unreachableNodeCCtx
+localCallNodes
+postIsNotPre
+postHasUniquePre
+uniquePostUpdate
+postIsInSameCallable
+reverseRead
+argHasPostUpdate
+postWithInFlow
+viableImplInCallContextTooLarge
+uniqueParameterNodeAtPosition
+uniqueParameterNodePosition
+uniqueContentApprox

python/ql/test/experimental/dataflow/exceptions/dataflow-consistency.ql

@@ -0,0 +1 @@
+import semmle.python.dataflow.new.internal.DataFlowImplConsistency::Consistency

python/ql/test/experimental/dataflow/exceptions/test.py

@@ -0,0 +1,35 @@
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.dirname((__file__))))
+from testlib import expects
+
+# These are defined so that we can evaluate the test code.
+NONSOURCE = "not a source"
+SOURCE = "source"
+
+
+def is_source(x):
+    return x == "source" or x == b"source" or x == 42 or x == 42.0 or x == 42j
+
+
+def SINK(x):
+    if is_source(x):
+        print("OK")
+    else:
+        print("Unexpected flow", x)
+
+
+def SINK_F(x):
+    if is_source(x):
+        print("Unexpected flow", x)
+    else:
+        print("OK")
+
+def test_as_binding():
+    try:
+        e_with_source = Exception()
+        e_with_source.a = SOURCE
+        raise e_with_source
+    except Exception as e:
+        SINK(e.a) # $ MISSING: flow

python/ql/test/experimental/dataflow/exceptions/test_group.py

@@ -0,0 +1,77 @@
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.dirname((__file__))))
+from testlib import expects
+
+# These are defined so that we can evaluate the test code.
+NONSOURCE = "not a source"
+SOURCE = "source"
+
+
+def is_source(x):
+    return x == "source" or x == b"source" or x == 42 or x == 42.0 or x == 42j
+
+
+def SINK(x):
+    if is_source(x):
+        print("OK")
+    else:
+        print("Unexpected flow", x)
+
+
+def SINK_F(x):
+    if is_source(x):
+        print("Unexpected flow", x)
+    else:
+        print("OK")
+
+def test_as_binding():
+    try:
+        e_with_source = Exception()
+        e_with_source.a = SOURCE
+        raise e_with_source
+    except* Exception as eg:
+        SINK(eg.exceptions[0].a) # $ MISSING: flow
+
+@expects(4)
+def test_group():
+    value_error_with_source = ValueError()
+    value_error_with_source.a = SOURCE
+
+    type_error_without_source = TypeError()
+    type_error_without_source.a = NONSOURCE
+
+    os_error_without_source = OSError()
+    os_error_without_source.a = NONSOURCE
+
+    eg = ExceptionGroup(
+        "one",
+        [
+            type_error_without_source,
+            ExceptionGroup(
+                "two",
+                    [type_error_without_source, value_error_with_source]
+            ),
+            ExceptionGroup(
+                    "three",
+                    [os_error_without_source]
+            )
+        ]
+    )
+    try:
+        raise eg
+    except* (TypeError, OSError) as es:
+        # The matched sub-group, represented by `es` is filtered,
+        # but the nesting is in place.
+        SINK_F(es.exceptions[0].a)
+        SINK_F(es.exceptions[1].exceptions[0].a)
+        SINK_F(es.exceptions[2].exceptions[0].a)
+    except* ValueError as es:
+        # The matched sub-group, represented by `es` is filtered,
+        # but the nesting is in place.
+        # So the ValueError was originally found in
+        # `eg.exceptions[1].exceptions[1].a`
+        # but now it is found in
+        # `es.exceptions[0].exceptions[0].a`
+        SINK(es.exceptions[0].exceptions[0].a) # $ MISSING: flow

python/ql/test/experimental/dataflow/validTest.py

@@ -51,6 +51,14 @@ def check_tests_valid(testFile):
                 check_async_test_function(item)
 
 
+def check_tests_valid_after_version(testFile, version):
+
+    if sys.version_info[:2] >= version:
+        print("INFO: Will run tests in", testFile, "since we're running Python", version, "or newer")
+        check_tests_valid(testFile)
+    else:
+        print("WARN: Will not run tests in", testFile, "since we're running Python", sys.version_info[:2], "and need", version, "or newer")
+
 if __name__ == "__main__":
     check_tests_valid("coverage.classes")
     check_tests_valid("coverage.test")
@@ -61,12 +69,9 @@ if __name__ == "__main__":
     check_tests_valid("variable-capture.dict")
     check_tests_valid("module-initialization.multiphase")
     check_tests_valid("fieldflow.test")
-
-    if sys.version_info[:2] >= (3, 10):
-        print("INFO: Will run `match` tests since we're running Python 3.10 or newer")
-        check_tests_valid("match.test")
-    else:
-        print("WARN: Skipping `match` tests since we're not running 3.10 or newer")
+    check_tests_valid_after_version("match.test", (3, 10))
+    check_tests_valid("exceptions.test")
+    check_tests_valid_after_version("exceptions.test_group", (3, 11))
 
     # The below fails when trying to import modules
     # check_tests_valid("module-initialization.test")

python/ql/test/query-tests/analysis/suppression/AlertSuppression.expected

@@ -14,7 +14,11 @@
 | test.py:18:4:18:12 | Comment #    lgtm |     lgtm | lgtm | test.py:18:1:18:12 | suppression range |
 | test.py:19:4:19:31 | Comment # lgtm    [py/line-too-long] |  lgtm    [py/line-too-long] | lgtm    [py/line-too-long] | test.py:19:1:19:31 | suppression range |
 | test.py:20:4:20:14 | Comment # lgtm lgtm |  lgtm lgtm | lgtm | test.py:20:1:20:14 | suppression range |
+| test.py:23:1:23:41 | Comment #lgtm -- Ignore this -- No line or scope. | lgtm -- Ignore this -- No line or scope. | lgtm | test.py:23:1:23:41 | suppression range |
+| test.py:23:1:23:41 | Comment #lgtm -- Ignore this -- No line or scope. | lgtm -- Ignore this -- No line or scope. | lgtm | test.py:24:0:24:0 | suppression range |
 | test.py:27:12:27:23 | Comment #lgtm [func] | lgtm [func] | lgtm [func] | test.py:27:1:27:23 | suppression range |
+| test.py:28:5:28:70 | Comment # lgtm -- Blank line (ignore for now, maybe scope wide in future). |  lgtm -- Blank line (ignore for now, maybe scope wide in future). | lgtm | test.py:28:1:28:70 | suppression range |
+| test.py:28:5:28:70 | Comment # lgtm -- Blank line (ignore for now, maybe scope wide in future). |  lgtm -- Blank line (ignore for now, maybe scope wide in future). | lgtm | test.py:29:0:29:0 | suppression range |
 | test.py:29:17:29:35 | Comment # lgtm on docstring |  lgtm on docstring | lgtm | test.py:29:1:29:35 | suppression range |
 | test.py:30:16:30:47 | Comment #lgtm [py/duplicate-key-in-dict] | lgtm [py/duplicate-key-in-dict] | lgtm [py/duplicate-key-in-dict] | test.py:30:1:30:47 | suppression range |
 | test.py:35:10:35:21 | Comment # lgtm class |  lgtm class | lgtm | test.py:35:1:35:21 | suppression range |
@@ -22,6 +26,8 @@
 | test.py:39:4:39:8 | Comment #noqa | noqa | lgtm | test.py:39:1:39:8 | suppression range |
 | test.py:40:4:40:9 | Comment # noqa |  noqa | lgtm | test.py:40:1:40:9 | suppression range |
 | test.py:45:4:45:31 | Comment # noqa -- Some extra detail. |  noqa -- Some extra detail. | lgtm | test.py:45:1:45:31 | suppression range |
+| test.py:49:1:49:10 | Comment #LGTM-1929 | LGTM-1929 | LGTM | test.py:49:1:49:10 | suppression range |
+| test.py:49:1:49:10 | Comment #LGTM-1929 | LGTM-1929 | LGTM | test.py:50:0:50:0 | suppression range |
 | test.py:50:34:50:117 | Comment # noqa: E501; (line too long) pylint: disable=invalid-name; lgtm [py/missing-equals] |  noqa: E501; (line too long) pylint: disable=invalid-name; lgtm [py/missing-equals] | lgtm [py/missing-equals] | test.py:50:1:50:117 | suppression range |
 | test.py:52:4:52:67 | Comment # noqa: E501; (line too long) pylint: disable=invalid-name; lgtm |  noqa: E501; (line too long) pylint: disable=invalid-name; lgtm | lgtm | test.py:52:1:52:67 | suppression range |
 | test.py:53:4:53:78 | Comment # random nonsense lgtm [py/missing-equals] and then some more commentary... |  random nonsense lgtm [py/missing-equals] and then some more commentary... | lgtm [py/missing-equals] | test.py:53:1:53:78 | suppression range |
@@ -31,6 +37,9 @@
 | test.py:65:4:65:60 | Comment # lgtm[py/line-too-long] and lgtm[py/non-callable-called] |  lgtm[py/line-too-long] and lgtm[py/non-callable-called] | lgtm[py/non-callable-called] | test.py:65:1:65:60 | suppression range |
 | test.py:66:4:66:33 | Comment # lgtm[py/line-too-long]; lgtm |  lgtm[py/line-too-long]; lgtm | lgtm | test.py:66:1:66:33 | suppression range |
 | test.py:66:4:66:33 | Comment # lgtm[py/line-too-long]; lgtm |  lgtm[py/line-too-long]; lgtm | lgtm[py/line-too-long] | test.py:66:1:66:33 | suppression range |
+| test.py:69:1:69:26 | Comment # codeql[py/line-too-long] |  codeql[py/line-too-long] | lgtm[py/line-too-long] | test.py:70:0:70:0 | suppression range |
+| test.py:71:1:71:25 | Comment #CODEQL[py/line-too-long] | CODEQL[py/line-too-long] | lgtm[py/line-too-long] | test.py:72:0:72:0 | suppression range |
+| test.py:73:1:73:63 | Comment # codeql[py/line-too-long] -- because I know better than codeql |  codeql[py/line-too-long] -- because I know better than codeql | lgtm[py/line-too-long] | test.py:74:0:74:0 | suppression range |
 | testWindows.py:4:4:4:9 | Comment # lgtm |  lgtm | lgtm | testWindows.py:4:1:4:9 | suppression range |
 | testWindows.py:5:4:5:27 | Comment # lgtm[py/line-too-long] |  lgtm[py/line-too-long] | lgtm[py/line-too-long] | testWindows.py:5:1:5:27 | suppression range |
 | testWindows.py:6:4:6:51 | Comment # lgtm[py/line-too-long, py/non-callable-called] |  lgtm[py/line-too-long, py/non-callable-called] | lgtm[py/line-too-long, py/non-callable-called] | testWindows.py:6:1:6:51 | suppression range |
@@ -47,14 +56,22 @@
 | testWindows.py:18:4:18:12 | Comment #    lgtm |     lgtm | lgtm | testWindows.py:18:1:18:12 | suppression range |
 | testWindows.py:19:4:19:31 | Comment # lgtm    [py/line-too-long] |  lgtm    [py/line-too-long] | lgtm    [py/line-too-long] | testWindows.py:19:1:19:31 | suppression range |
 | testWindows.py:20:4:20:14 | Comment # lgtm lgtm |  lgtm lgtm | lgtm | testWindows.py:20:1:20:14 | suppression range |
+| testWindows.py:23:1:23:41 | Comment #lgtm -- Ignore this -- No line or scope. | lgtm -- Ignore this -- No line or scope. | lgtm | testWindows.py:23:1:23:41 | suppression range |
+| testWindows.py:23:1:23:41 | Comment #lgtm -- Ignore this -- No line or scope. | lgtm -- Ignore this -- No line or scope. | lgtm | testWindows.py:24:0:24:0 | suppression range |
 | testWindows.py:27:12:27:23 | Comment #lgtm [func] | lgtm [func] | lgtm [func] | testWindows.py:27:1:27:23 | suppression range |
+| testWindows.py:28:5:28:70 | Comment # lgtm -- Blank line (ignore for now, maybe scope wide in future). |  lgtm -- Blank line (ignore for now, maybe scope wide in future). | lgtm | testWindows.py:28:1:28:70 | suppression range |
+| testWindows.py:28:5:28:70 | Comment # lgtm -- Blank line (ignore for now, maybe scope wide in future). |  lgtm -- Blank line (ignore for now, maybe scope wide in future). | lgtm | testWindows.py:29:0:29:0 | suppression range |
 | testWindows.py:29:17:29:35 | Comment # lgtm on docstring |  lgtm on docstring | lgtm | testWindows.py:29:1:29:35 | suppression range |
 | testWindows.py:30:16:30:47 | Comment #lgtm [py/duplicate-key-in-dict] | lgtm [py/duplicate-key-in-dict] | lgtm [py/duplicate-key-in-dict] | testWindows.py:30:1:30:47 | suppression range |
 | testWindows.py:35:10:35:21 | Comment # lgtm class |  lgtm class | lgtm | testWindows.py:35:1:35:21 | suppression range |
 | testWindows.py:36:21:36:33 | Comment # lgtm method |  lgtm method | lgtm | testWindows.py:36:1:36:33 | suppression range |
 | testWindows.py:39:3:39:7 | Comment #noqa | noqa | lgtm | testWindows.py:39:1:39:7 | suppression range |
 | testWindows.py:40:4:40:9 | Comment # noqa |  noqa | lgtm | testWindows.py:40:1:40:9 | suppression range |
+| testWindows.py:45:1:45:28 | Comment # noqa -- Some extra detail. |  noqa -- Some extra detail. | lgtm | testWindows.py:45:1:45:28 | suppression range |
 | testWindows.py:48:4:48:60 | Comment # lgtm[py/line-too-long] and lgtm[py/non-callable-called] |  lgtm[py/line-too-long] and lgtm[py/non-callable-called] | lgtm[py/line-too-long] | testWindows.py:48:1:48:60 | suppression range |
 | testWindows.py:48:4:48:60 | Comment # lgtm[py/line-too-long] and lgtm[py/non-callable-called] |  lgtm[py/line-too-long] and lgtm[py/non-callable-called] | lgtm[py/non-callable-called] | testWindows.py:48:1:48:60 | suppression range |
 | testWindows.py:49:4:49:33 | Comment # lgtm[py/line-too-long]; lgtm |  lgtm[py/line-too-long]; lgtm | lgtm | testWindows.py:49:1:49:33 | suppression range |
 | testWindows.py:49:4:49:33 | Comment # lgtm[py/line-too-long]; lgtm |  lgtm[py/line-too-long]; lgtm | lgtm[py/line-too-long] | testWindows.py:49:1:49:33 | suppression range |
+| testWindows.py:52:1:52:26 | Comment # codeql[py/line-too-long] |  codeql[py/line-too-long] | lgtm[py/line-too-long] | testWindows.py:53:0:53:0 | suppression range |
+| testWindows.py:54:1:54:25 | Comment #CODEQL[py/line-too-long] | CODEQL[py/line-too-long] | lgtm[py/line-too-long] | testWindows.py:55:0:55:0 | suppression range |
+| testWindows.py:56:1:56:63 | Comment # codeql[py/line-too-long] -- because I know better than codeql |  codeql[py/line-too-long] -- because I know better than codeql | lgtm[py/line-too-long] | testWindows.py:57:0:57:0 | suppression range |

python/ql/test/query-tests/analysis/suppression/AlertSuppression.qlref

@@ -1 +1 @@
-analysis/AlertSuppression.ql
+AlertSuppression.ql

python/ql/test/query-tests/analysis/suppression/test.py

@@ -64,3 +64,12 @@ class frozenbidict(BidictBase):  # noqa: E501; (line too long) pylint: disable=i
 
 "" # lgtm[py/line-too-long] and lgtm[py/non-callable-called]
 "" # lgtm[py/line-too-long]; lgtm
+
+#CodeQL comments
+# codeql[py/line-too-long]
+""
+#CODEQL[py/line-too-long]
+""
+# codeql[py/line-too-long] -- because I know better than codeql
+""
+"" # codeql[py/line-too-long]

python/ql/test/query-tests/analysis/suppression/testWindows.py

@@ -47,3 +47,12 @@ class C: # lgtm class
 
 "" # lgtm[py/line-too-long] and lgtm[py/non-callable-called]
 "" # lgtm[py/line-too-long]; lgtm
+
+#CodeQL comments
+# codeql[py/line-too-long]
+""
+#CODEQL[py/line-too-long]
+""
+# codeql[py/line-too-long] -- because I know better than codeql
+""
+"" # codeql[py/line-too-long]