Merge branch 'main' of github.com:github/codeql into python-add-parameter-default-value-flow-step

python/ql/lib/semmle/python/RegexTreeView.qll

@@ -7,6 +7,10 @@ private import semmle.python.regex
  * An element containing a regular expression term, that is, either
  * a string literal (parsed as a regular expression)
  * or another regular expression term.
+ *
+ * For sequences and alternations, we require at least one child.
+ * Otherwise, we wish to represent the term differently.
+ * This avoids multiple representations of the same term.
  */
 newtype TRegExpParent =
   /** A string literal used as a regular expression */
@@ -14,9 +18,18 @@ newtype TRegExpParent =
   /** A quantified term */
   TRegExpQuantifier(Regex re, int start, int end) { re.qualifiedItem(start, end, _, _) } or
   /** A sequence term */
-  TRegExpSequence(Regex re, int start, int end) { re.sequence(start, end) } or
-  /** An alternatio term */
-  TRegExpAlt(Regex re, int start, int end) { re.alternation(start, end) } or
+  TRegExpSequence(Regex re, int start, int end) {
+    re.sequence(start, end) and
+    exists(seqChild(re, start, end, 1)) // if a sequence does not have more than one element, it should be treated as that element instead.
+  } or
+  /** An alternation term */
+  TRegExpAlt(Regex re, int start, int end) {
+    re.alternation(start, end) and
+    exists(int part_end |
+      re.alternationOption(start, end, start, part_end) and
+      part_end < end
+    ) // if an alternation does not have more than one element, it should be treated as that element instead.
+  } or
   /** A character class term */
   TRegExpCharacterClass(Regex re, int start, int end) { re.charSet(start, end) } or
   /** A character range term */
@@ -93,8 +106,7 @@ class RegExpTerm extends RegExpParent {
     or
     this = TRegExpQuantifier(re, start, end)
     or
-    this = TRegExpSequence(re, start, end) and
-    exists(seqChild(re, start, end, 1)) // if a sequence does not have more than one element, it should be treated as that element instead.
+    this = TRegExpSequence(re, start, end)
     or
     this = TRegExpSpecialChar(re, start, end)
   }
@@ -341,10 +353,7 @@ class RegExpRange extends RegExpQuantifier {
  * This is a sequence with the elements `(ECMA|Java)` and `Script`.
  */
 class RegExpSequence extends RegExpTerm, TRegExpSequence {
-  RegExpSequence() {
-    this = TRegExpSequence(re, start, end) and
-    exists(seqChild(re, start, end, 1)) // if a sequence does not have more than one element, it should be treated as that element instead.
-  }
+  RegExpSequence() { this = TRegExpSequence(re, start, end) }
 
   override RegExpTerm getChild(int i) { result = seqChild(re, start, end, i) }
 

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll

@@ -3258,24 +3258,16 @@ class PathNode extends TPathNode {
   /** Gets the associated configuration. */
   Configuration getConfiguration() { none() }
 
-  private predicate isHidden() {
-    hiddenNode(this.(PathNodeImpl).getNodeEx().asNode()) and
-    not this.isSource() and
-    not this instanceof PathNodeSink
-    or
-    this.(PathNodeImpl).getNodeEx() instanceof TNodeImplicitRead
-  }
-
   private PathNode getASuccessorIfHidden() {
-    this.isHidden() and
+    this.(PathNodeImpl).isHidden() and
     result = this.(PathNodeImpl).getASuccessorImpl()
   }
 
   /** Gets a successor of this node, if any. */
   final PathNode getASuccessor() {
     result = this.(PathNodeImpl).getASuccessorImpl().getASuccessorIfHidden*() and
-    not this.isHidden() and
-    not result.isHidden()
+    not this.(PathNodeImpl).isHidden() and
+    not result.(PathNodeImpl).isHidden()
   }
 
   /** Holds if this node is a source. */
@@ -3287,6 +3279,14 @@ abstract private class PathNodeImpl extends PathNode {
 
   abstract NodeEx getNodeEx();
 
+  predicate isHidden() {
+    hiddenNode(this.getNodeEx().asNode()) and
+    not this.isSource() and
+    not this instanceof PathNodeSink
+    or
+    this.getNodeEx() instanceof TNodeImplicitRead
+  }
+
   private string ppAp() {
     this instanceof PathNodeSink and result = ""
     or
@@ -3313,9 +3313,14 @@ abstract private class PathNodeImpl extends PathNode {
 }
 
 /** Holds if `n` can reach a sink. */
-private predicate reach(PathNode n) { n instanceof PathNodeSink or reach(n.getASuccessor()) }
+private predicate directReach(PathNode n) {
+  n instanceof PathNodeSink or directReach(n.getASuccessor())
+}
 
-/** Holds if `n1.getSucc() = n2` and `n2` can reach a sink. */
+/** Holds if `n` can reach a sink or is used in a subpath. */
+private predicate reach(PathNode n) { directReach(n) or Subpaths::retReach(n) }
+
+/** Holds if `n1.getSucc() = n2` and `n2` can reach a sink or is used in a subpath. */
 private predicate pathSucc(PathNode n1, PathNode n2) { n1.getASuccessor() = n2 and reach(n2) }
 
 private predicate pathSuccPlus(PathNode n1, PathNode n2) = fastTC(pathSucc/2)(n1, n2)
@@ -3331,6 +3336,8 @@ module PathGraph {
   query predicate nodes(PathNode n, string key, string val) {
     reach(n) and key = "semmle.label" and val = n.toString()
   }
+
+  query predicate subpaths = Subpaths::subpaths/4;
 }
 
 /**
@@ -3622,6 +3629,86 @@ private predicate pathThroughCallable(PathNodeMid mid, NodeEx out, CallContext c
   )
 }
 
+private module Subpaths {
+  /**
+   * Holds if `(arg, par, ret, out)` forms a subpath-tuple and `ret` is determined by
+   * `kind`, `sc`, `apout`, and `innercc`.
+   */
+  pragma[nomagic]
+  private predicate subpaths01(
+    PathNode arg, ParamNodeEx par, SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind,
+    NodeEx out, AccessPath apout
+  ) {
+    pathThroughCallable(arg, out, _, apout) and
+    pathIntoCallable(arg, par, _, innercc, sc, _) and
+    paramFlowsThrough(kind, innercc, sc, apout, _, unbindConf(arg.getConfiguration()))
+  }
+
+  /**
+   * Holds if `(arg, par, ret, out)` forms a subpath-tuple and `ret` is determined by
+   * `kind`, `sc`, `apout`, and `innercc`.
+   */
+  pragma[nomagic]
+  private predicate subpaths02(
+    PathNode arg, ParamNodeEx par, SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind,
+    NodeEx out, AccessPath apout
+  ) {
+    subpaths01(arg, par, sc, innercc, kind, out, apout) and
+    out.asNode() = kind.getAnOutNode(_)
+  }
+
+  pragma[nomagic]
+  private Configuration getPathNodeConf(PathNode n) { result = n.getConfiguration() }
+
+  /**
+   * Holds if `(arg, par, ret, out)` forms a subpath-tuple.
+   */
+  pragma[nomagic]
+  private predicate subpaths03(
+    PathNode arg, ParamNodeEx par, PathNodeMid ret, NodeEx out, AccessPath apout
+  ) {
+    exists(SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind, RetNodeEx retnode |
+      subpaths02(arg, par, sc, innercc, kind, out, apout) and
+      ret.getNodeEx() = retnode and
+      kind = retnode.getKind() and
+      innercc = ret.getCallContext() and
+      sc = ret.getSummaryCtx() and
+      ret.getConfiguration() = unbindConf(getPathNodeConf(arg)) and
+      apout = ret.getAp() and
+      not ret.isHidden()
+    )
+  }
+
+  /**
+   * Holds if `(arg, par, ret, out)` forms a subpath-tuple, that is, flow through
+   * a subpath between `par` and `ret` with the connecting edges `arg -> par` and
+   * `ret -> out` is summarized as the edge `arg -> out`.
+   */
+  predicate subpaths(PathNode arg, PathNodeImpl par, PathNodeMid ret, PathNodeMid out) {
+    exists(ParamNodeEx p, NodeEx o, AccessPath apout |
+      arg.getASuccessor() = par and
+      arg.getASuccessor() = out and
+      subpaths03(arg, p, ret, o, apout) and
+      par.getNodeEx() = p and
+      out.getNodeEx() = o and
+      out.getAp() = apout
+    )
+  }
+
+  /**
+   * Holds if `n` can reach a return node in a summarized subpath.
+   */
+  predicate retReach(PathNode n) {
+    subpaths(_, _, n, _)
+    or
+    exists(PathNode mid |
+      retReach(mid) and
+      n.getASuccessor() = mid and
+      not subpaths(_, mid, _, _)
+    )
+  }
+}
+
 /**
  * Holds if data can flow (inter-procedurally) from `source` to `sink`.
  *

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll

@@ -3258,24 +3258,16 @@ class PathNode extends TPathNode {
   /** Gets the associated configuration. */
   Configuration getConfiguration() { none() }
 
-  private predicate isHidden() {
-    hiddenNode(this.(PathNodeImpl).getNodeEx().asNode()) and
-    not this.isSource() and
-    not this instanceof PathNodeSink
-    or
-    this.(PathNodeImpl).getNodeEx() instanceof TNodeImplicitRead
-  }
-
   private PathNode getASuccessorIfHidden() {
-    this.isHidden() and
+    this.(PathNodeImpl).isHidden() and
     result = this.(PathNodeImpl).getASuccessorImpl()
   }
 
   /** Gets a successor of this node, if any. */
   final PathNode getASuccessor() {
     result = this.(PathNodeImpl).getASuccessorImpl().getASuccessorIfHidden*() and
-    not this.isHidden() and
-    not result.isHidden()
+    not this.(PathNodeImpl).isHidden() and
+    not result.(PathNodeImpl).isHidden()
   }
 
   /** Holds if this node is a source. */
@@ -3287,6 +3279,14 @@ abstract private class PathNodeImpl extends PathNode {
 
   abstract NodeEx getNodeEx();
 
+  predicate isHidden() {
+    hiddenNode(this.getNodeEx().asNode()) and
+    not this.isSource() and
+    not this instanceof PathNodeSink
+    or
+    this.getNodeEx() instanceof TNodeImplicitRead
+  }
+
   private string ppAp() {
     this instanceof PathNodeSink and result = ""
     or
@@ -3313,9 +3313,14 @@ abstract private class PathNodeImpl extends PathNode {
 }
 
 /** Holds if `n` can reach a sink. */
-private predicate reach(PathNode n) { n instanceof PathNodeSink or reach(n.getASuccessor()) }
+private predicate directReach(PathNode n) {
+  n instanceof PathNodeSink or directReach(n.getASuccessor())
+}
 
-/** Holds if `n1.getSucc() = n2` and `n2` can reach a sink. */
+/** Holds if `n` can reach a sink or is used in a subpath. */
+private predicate reach(PathNode n) { directReach(n) or Subpaths::retReach(n) }
+
+/** Holds if `n1.getSucc() = n2` and `n2` can reach a sink or is used in a subpath. */
 private predicate pathSucc(PathNode n1, PathNode n2) { n1.getASuccessor() = n2 and reach(n2) }
 
 private predicate pathSuccPlus(PathNode n1, PathNode n2) = fastTC(pathSucc/2)(n1, n2)
@@ -3331,6 +3336,8 @@ module PathGraph {
   query predicate nodes(PathNode n, string key, string val) {
     reach(n) and key = "semmle.label" and val = n.toString()
   }
+
+  query predicate subpaths = Subpaths::subpaths/4;
 }
 
 /**
@@ -3622,6 +3629,86 @@ private predicate pathThroughCallable(PathNodeMid mid, NodeEx out, CallContext c
   )
 }
 
+private module Subpaths {
+  /**
+   * Holds if `(arg, par, ret, out)` forms a subpath-tuple and `ret` is determined by
+   * `kind`, `sc`, `apout`, and `innercc`.
+   */
+  pragma[nomagic]
+  private predicate subpaths01(
+    PathNode arg, ParamNodeEx par, SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind,
+    NodeEx out, AccessPath apout
+  ) {
+    pathThroughCallable(arg, out, _, apout) and
+    pathIntoCallable(arg, par, _, innercc, sc, _) and
+    paramFlowsThrough(kind, innercc, sc, apout, _, unbindConf(arg.getConfiguration()))
+  }
+
+  /**
+   * Holds if `(arg, par, ret, out)` forms a subpath-tuple and `ret` is determined by
+   * `kind`, `sc`, `apout`, and `innercc`.
+   */
+  pragma[nomagic]
+  private predicate subpaths02(
+    PathNode arg, ParamNodeEx par, SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind,
+    NodeEx out, AccessPath apout
+  ) {
+    subpaths01(arg, par, sc, innercc, kind, out, apout) and
+    out.asNode() = kind.getAnOutNode(_)
+  }
+
+  pragma[nomagic]
+  private Configuration getPathNodeConf(PathNode n) { result = n.getConfiguration() }
+
+  /**
+   * Holds if `(arg, par, ret, out)` forms a subpath-tuple.
+   */
+  pragma[nomagic]
+  private predicate subpaths03(
+    PathNode arg, ParamNodeEx par, PathNodeMid ret, NodeEx out, AccessPath apout
+  ) {
+    exists(SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind, RetNodeEx retnode |
+      subpaths02(arg, par, sc, innercc, kind, out, apout) and
+      ret.getNodeEx() = retnode and
+      kind = retnode.getKind() and
+      innercc = ret.getCallContext() and
+      sc = ret.getSummaryCtx() and
+      ret.getConfiguration() = unbindConf(getPathNodeConf(arg)) and
+      apout = ret.getAp() and
+      not ret.isHidden()
+    )
+  }
+
+  /**
+   * Holds if `(arg, par, ret, out)` forms a subpath-tuple, that is, flow through
+   * a subpath between `par` and `ret` with the connecting edges `arg -> par` and
+   * `ret -> out` is summarized as the edge `arg -> out`.
+   */
+  predicate subpaths(PathNode arg, PathNodeImpl par, PathNodeMid ret, PathNodeMid out) {
+    exists(ParamNodeEx p, NodeEx o, AccessPath apout |
+      arg.getASuccessor() = par and
+      arg.getASuccessor() = out and
+      subpaths03(arg, p, ret, o, apout) and
+      par.getNodeEx() = p and
+      out.getNodeEx() = o and
+      out.getAp() = apout
+    )
+  }
+
+  /**
+   * Holds if `n` can reach a return node in a summarized subpath.
+   */
+  predicate retReach(PathNode n) {
+    subpaths(_, _, n, _)
+    or
+    exists(PathNode mid |
+      retReach(mid) and
+      n.getASuccessor() = mid and
+      not subpaths(_, mid, _, _)
+    )
+  }
+}
+
 /**
  * Holds if data can flow (inter-procedurally) from `source` to `sink`.
  *

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll

@@ -3258,24 +3258,16 @@ class PathNode extends TPathNode {
   /** Gets the associated configuration. */
   Configuration getConfiguration() { none() }
 
-  private predicate isHidden() {
-    hiddenNode(this.(PathNodeImpl).getNodeEx().asNode()) and
-    not this.isSource() and
-    not this instanceof PathNodeSink
-    or
-    this.(PathNodeImpl).getNodeEx() instanceof TNodeImplicitRead
-  }
-
   private PathNode getASuccessorIfHidden() {
-    this.isHidden() and
+    this.(PathNodeImpl).isHidden() and
     result = this.(PathNodeImpl).getASuccessorImpl()
   }
 
   /** Gets a successor of this node, if any. */
   final PathNode getASuccessor() {
     result = this.(PathNodeImpl).getASuccessorImpl().getASuccessorIfHidden*() and
-    not this.isHidden() and
-    not result.isHidden()
+    not this.(PathNodeImpl).isHidden() and
+    not result.(PathNodeImpl).isHidden()
   }
 
   /** Holds if this node is a source. */
@@ -3287,6 +3279,14 @@ abstract private class PathNodeImpl extends PathNode {
 
   abstract NodeEx getNodeEx();
 
+  predicate isHidden() {
+    hiddenNode(this.getNodeEx().asNode()) and
+    not this.isSource() and
+    not this instanceof PathNodeSink
+    or
+    this.getNodeEx() instanceof TNodeImplicitRead
+  }
+
   private string ppAp() {
     this instanceof PathNodeSink and result = ""
     or
@@ -3313,9 +3313,14 @@ abstract private class PathNodeImpl extends PathNode {
 }
 
 /** Holds if `n` can reach a sink. */
-private predicate reach(PathNode n) { n instanceof PathNodeSink or reach(n.getASuccessor()) }
+private predicate directReach(PathNode n) {
+  n instanceof PathNodeSink or directReach(n.getASuccessor())
+}
 
-/** Holds if `n1.getSucc() = n2` and `n2` can reach a sink. */
+/** Holds if `n` can reach a sink or is used in a subpath. */
+private predicate reach(PathNode n) { directReach(n) or Subpaths::retReach(n) }
+
+/** Holds if `n1.getSucc() = n2` and `n2` can reach a sink or is used in a subpath. */
 private predicate pathSucc(PathNode n1, PathNode n2) { n1.getASuccessor() = n2 and reach(n2) }
 
 private predicate pathSuccPlus(PathNode n1, PathNode n2) = fastTC(pathSucc/2)(n1, n2)
@@ -3331,6 +3336,8 @@ module PathGraph {
   query predicate nodes(PathNode n, string key, string val) {
     reach(n) and key = "semmle.label" and val = n.toString()
   }
+
+  query predicate subpaths = Subpaths::subpaths/4;
 }
 
 /**
@@ -3622,6 +3629,86 @@ private predicate pathThroughCallable(PathNodeMid mid, NodeEx out, CallContext c
   )
 }
 
+private module Subpaths {
+  /**
+   * Holds if `(arg, par, ret, out)` forms a subpath-tuple and `ret` is determined by
+   * `kind`, `sc`, `apout`, and `innercc`.
+   */
+  pragma[nomagic]
+  private predicate subpaths01(
+    PathNode arg, ParamNodeEx par, SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind,
+    NodeEx out, AccessPath apout
+  ) {
+    pathThroughCallable(arg, out, _, apout) and
+    pathIntoCallable(arg, par, _, innercc, sc, _) and
+    paramFlowsThrough(kind, innercc, sc, apout, _, unbindConf(arg.getConfiguration()))
+  }
+
+  /**
+   * Holds if `(arg, par, ret, out)` forms a subpath-tuple and `ret` is determined by
+   * `kind`, `sc`, `apout`, and `innercc`.
+   */
+  pragma[nomagic]
+  private predicate subpaths02(
+    PathNode arg, ParamNodeEx par, SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind,
+    NodeEx out, AccessPath apout
+  ) {
+    subpaths01(arg, par, sc, innercc, kind, out, apout) and
+    out.asNode() = kind.getAnOutNode(_)
+  }
+
+  pragma[nomagic]
+  private Configuration getPathNodeConf(PathNode n) { result = n.getConfiguration() }
+
+  /**
+   * Holds if `(arg, par, ret, out)` forms a subpath-tuple.
+   */
+  pragma[nomagic]
+  private predicate subpaths03(
+    PathNode arg, ParamNodeEx par, PathNodeMid ret, NodeEx out, AccessPath apout
+  ) {
+    exists(SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind, RetNodeEx retnode |
+      subpaths02(arg, par, sc, innercc, kind, out, apout) and
+      ret.getNodeEx() = retnode and
+      kind = retnode.getKind() and
+      innercc = ret.getCallContext() and
+      sc = ret.getSummaryCtx() and
+      ret.getConfiguration() = unbindConf(getPathNodeConf(arg)) and
+      apout = ret.getAp() and
+      not ret.isHidden()
+    )
+  }
+
+  /**
+   * Holds if `(arg, par, ret, out)` forms a subpath-tuple, that is, flow through
+   * a subpath between `par` and `ret` with the connecting edges `arg -> par` and
+   * `ret -> out` is summarized as the edge `arg -> out`.
+   */
+  predicate subpaths(PathNode arg, PathNodeImpl par, PathNodeMid ret, PathNodeMid out) {
+    exists(ParamNodeEx p, NodeEx o, AccessPath apout |
+      arg.getASuccessor() = par and
+      arg.getASuccessor() = out and
+      subpaths03(arg, p, ret, o, apout) and
+      par.getNodeEx() = p and
+      out.getNodeEx() = o and
+      out.getAp() = apout
+    )
+  }
+
+  /**
+   * Holds if `n` can reach a return node in a summarized subpath.
+   */
+  predicate retReach(PathNode n) {
+    subpaths(_, _, n, _)
+    or
+    exists(PathNode mid |
+      retReach(mid) and
+      n.getASuccessor() = mid and
+      not subpaths(_, mid, _, _)
+    )
+  }
+}
+
 /**
  * Holds if data can flow (inter-procedurally) from `source` to `sink`.
  *

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll

@@ -3258,24 +3258,16 @@ class PathNode extends TPathNode {
   /** Gets the associated configuration. */
   Configuration getConfiguration() { none() }
 
-  private predicate isHidden() {
-    hiddenNode(this.(PathNodeImpl).getNodeEx().asNode()) and
-    not this.isSource() and
-    not this instanceof PathNodeSink
-    or
-    this.(PathNodeImpl).getNodeEx() instanceof TNodeImplicitRead
-  }
-
   private PathNode getASuccessorIfHidden() {
-    this.isHidden() and
+    this.(PathNodeImpl).isHidden() and
     result = this.(PathNodeImpl).getASuccessorImpl()
   }
 
   /** Gets a successor of this node, if any. */
   final PathNode getASuccessor() {
     result = this.(PathNodeImpl).getASuccessorImpl().getASuccessorIfHidden*() and
-    not this.isHidden() and
-    not result.isHidden()
+    not this.(PathNodeImpl).isHidden() and
+    not result.(PathNodeImpl).isHidden()
   }
 
   /** Holds if this node is a source. */
@@ -3287,6 +3279,14 @@ abstract private class PathNodeImpl extends PathNode {
 
   abstract NodeEx getNodeEx();
 
+  predicate isHidden() {
+    hiddenNode(this.getNodeEx().asNode()) and
+    not this.isSource() and
+    not this instanceof PathNodeSink
+    or
+    this.getNodeEx() instanceof TNodeImplicitRead
+  }
+
   private string ppAp() {
     this instanceof PathNodeSink and result = ""
     or
@@ -3313,9 +3313,14 @@ abstract private class PathNodeImpl extends PathNode {
 }
 
 /** Holds if `n` can reach a sink. */
-private predicate reach(PathNode n) { n instanceof PathNodeSink or reach(n.getASuccessor()) }
+private predicate directReach(PathNode n) {
+  n instanceof PathNodeSink or directReach(n.getASuccessor())
+}
 
-/** Holds if `n1.getSucc() = n2` and `n2` can reach a sink. */
+/** Holds if `n` can reach a sink or is used in a subpath. */
+private predicate reach(PathNode n) { directReach(n) or Subpaths::retReach(n) }
+
+/** Holds if `n1.getSucc() = n2` and `n2` can reach a sink or is used in a subpath. */
 private predicate pathSucc(PathNode n1, PathNode n2) { n1.getASuccessor() = n2 and reach(n2) }
 
 private predicate pathSuccPlus(PathNode n1, PathNode n2) = fastTC(pathSucc/2)(n1, n2)
@@ -3331,6 +3336,8 @@ module PathGraph {
   query predicate nodes(PathNode n, string key, string val) {
     reach(n) and key = "semmle.label" and val = n.toString()
   }
+
+  query predicate subpaths = Subpaths::subpaths/4;
 }
 
 /**
@@ -3622,6 +3629,86 @@ private predicate pathThroughCallable(PathNodeMid mid, NodeEx out, CallContext c
   )
 }
 
+private module Subpaths {
+  /**
+   * Holds if `(arg, par, ret, out)` forms a subpath-tuple and `ret` is determined by
+   * `kind`, `sc`, `apout`, and `innercc`.
+   */
+  pragma[nomagic]
+  private predicate subpaths01(
+    PathNode arg, ParamNodeEx par, SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind,
+    NodeEx out, AccessPath apout
+  ) {
+    pathThroughCallable(arg, out, _, apout) and
+    pathIntoCallable(arg, par, _, innercc, sc, _) and
+    paramFlowsThrough(kind, innercc, sc, apout, _, unbindConf(arg.getConfiguration()))
+  }
+
+  /**
+   * Holds if `(arg, par, ret, out)` forms a subpath-tuple and `ret` is determined by
+   * `kind`, `sc`, `apout`, and `innercc`.
+   */
+  pragma[nomagic]
+  private predicate subpaths02(
+    PathNode arg, ParamNodeEx par, SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind,
+    NodeEx out, AccessPath apout
+  ) {
+    subpaths01(arg, par, sc, innercc, kind, out, apout) and
+    out.asNode() = kind.getAnOutNode(_)
+  }
+
+  pragma[nomagic]
+  private Configuration getPathNodeConf(PathNode n) { result = n.getConfiguration() }
+
+  /**
+   * Holds if `(arg, par, ret, out)` forms a subpath-tuple.
+   */
+  pragma[nomagic]
+  private predicate subpaths03(
+    PathNode arg, ParamNodeEx par, PathNodeMid ret, NodeEx out, AccessPath apout
+  ) {
+    exists(SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind, RetNodeEx retnode |
+      subpaths02(arg, par, sc, innercc, kind, out, apout) and
+      ret.getNodeEx() = retnode and
+      kind = retnode.getKind() and
+      innercc = ret.getCallContext() and
+      sc = ret.getSummaryCtx() and
+      ret.getConfiguration() = unbindConf(getPathNodeConf(arg)) and
+      apout = ret.getAp() and
+      not ret.isHidden()
+    )
+  }
+
+  /**
+   * Holds if `(arg, par, ret, out)` forms a subpath-tuple, that is, flow through
+   * a subpath between `par` and `ret` with the connecting edges `arg -> par` and
+   * `ret -> out` is summarized as the edge `arg -> out`.
+   */
+  predicate subpaths(PathNode arg, PathNodeImpl par, PathNodeMid ret, PathNodeMid out) {
+    exists(ParamNodeEx p, NodeEx o, AccessPath apout |
+      arg.getASuccessor() = par and
+      arg.getASuccessor() = out and
+      subpaths03(arg, p, ret, o, apout) and
+      par.getNodeEx() = p and
+      out.getNodeEx() = o and
+      out.getAp() = apout
+    )
+  }
+
+  /**
+   * Holds if `n` can reach a return node in a summarized subpath.
+   */
+  predicate retReach(PathNode n) {
+    subpaths(_, _, n, _)
+    or
+    exists(PathNode mid |
+      retReach(mid) and
+      n.getASuccessor() = mid and
+      not subpaths(_, mid, _, _)
+    )
+  }
+}
+
 /**
  * Holds if data can flow (inter-procedurally) from `source` to `sink`.
  *

python/ql/lib/semmle/python/regex.qll

@@ -369,12 +369,12 @@ abstract class RegexString extends Expr {
       // hex value \xhh
       this.getChar(start + 1) = "x" and end = start + 4
       or
-      // octal value \ooo
+      // octal value \o, \oo, or \ooo
       end in [start + 2 .. start + 4] and
-      this.getText().substring(start + 1, end).toInt() >= 0 and
+      forall(int i | i in [start + 1 .. end - 1] | this.isOctal(i)) and
       not (
         end < start + 4 and
-        exists(this.getText().substring(start + 1, end + 1).toInt())
+        this.isOctal(end)
       )
       or
       // 16-bit hex value \uhhhh
@@ -392,6 +392,9 @@ abstract class RegexString extends Expr {
     )
   }
 
+  pragma[inline]
+  private predicate isOctal(int index) { this.getChar(index) = [0 .. 7].toString() }
+
   /** Holds if `index` is inside a character set. */
   predicate inCharSet(int index) {
     exists(int x, int y | this.charSet(x, y) and index in [x + 1 .. y - 2])
@@ -690,6 +693,7 @@ abstract class RegexString extends Expr {
 
   private predicate numbered_backreference(int start, int end, int value) {
     this.escapingChar(start) and
+    // starting with 0 makes it an octal escape
     not this.getChar(start + 1) = "0" and
     exists(string text, string svalue, int len |
       end = start + len and
@@ -698,8 +702,16 @@ abstract class RegexString extends Expr {
     |
       svalue = text.substring(start + 1, start + len) and
       value = svalue.toInt() and
-      not exists(text.substring(start + 1, start + len + 1).toInt()) and
-      value > 0
+      // value is composed of digits
+      forall(int i | i in [start + 1 .. start + len - 1] | this.getChar(i) = [0 .. 9].toString()) and
+      // a longer reference is not possible
+      not (
+        len = 2 and
+        exists(text.substring(start + 1, start + len + 1).toInt())
+      ) and
+      // 3 octal digits makes it an octal escape
+      not forall(int i | i in [start + 1 .. start + 4] | this.isOctal(i))
+      // TODO: Inside a character set, all numeric escapes are treated as characters.
     )
   }