Merge branch 'main' into import-refined

python/ql/lib/CHANGELOG.md

@@ -1,3 +1,22 @@
+## 0.8.2
+
+### New Features
+
+* Added support for merging two `PathGraph`s via disjoint union to allow results from multiple data flow computations in a single `path-problem` query.
+
+### Major Analysis Improvements
+
+* The main data flow and taint tracking APIs have been changed. The old APIs
+  remain in place for now and translate to the new through a
+  backwards-compatible wrapper. If multiple configurations are in scope
+  simultaneously, then this may affect results slightly. The new API is quite
+  similar to the old, but makes use of a configuration module instead of a
+  configuration class.
+
+### Minor Analysis Improvements
+
+* Deleted the deprecated `getPath` and `getFolder` predicates from the `XmlFile` class.
+
 ## 0.8.1
 
 ### Major Analysis Improvements

python/ql/lib/change-notes/2023-03-03-delete-deps.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* Deleted the deprecated `getPath` and `getFolder` predicates from the `XmlFile` class.

python/ql/lib/change-notes/2023-03-13-mergepathgraph.md

@@ -1,4 +0,0 @@
----
-category: feature
----
-* Added support for merging two `PathGraph`s via disjoint union to allow results from multiple data flow computations in a single `path-problem` query.

python/ql/lib/change-notes/released/0.8.2.md

@@ -1,9 +1,18 @@
----
-category: majorAnalysis
----
+## 0.8.2
+
+### New Features
+
+* Added support for merging two `PathGraph`s via disjoint union to allow results from multiple data flow computations in a single `path-problem` query.
+
+### Major Analysis Improvements
+
 * The main data flow and taint tracking APIs have been changed. The old APIs
   remain in place for now and translate to the new through a
   backwards-compatible wrapper. If multiple configurations are in scope
   simultaneously, then this may affect results slightly. The new API is quite
   similar to the old, but makes use of a configuration module instead of a
   configuration class.
+
+### Minor Analysis Improvements
+
+* Deleted the deprecated `getPath` and `getFolder` predicates from the `XmlFile` class.

python/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.8.1
+lastReleaseVersion: 0.8.2

python/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/python-all
-version: 0.8.2-dev
+version: 0.8.3-dev
 groups: python
 dbscheme: semmlecode.python.dbscheme
 extractor: python
@@ -7,6 +7,7 @@ library: true
 upgrades: upgrades
 dependencies:
   codeql/regex: ${workspace}
+  codeql/util: ${workspace}
   codeql/tutorial: ${workspace}
 dataExtensions:
   - semmle/python/frameworks/**/model.yml

python/ql/lib/semmle/python/RegexTreeView.qll

@@ -468,6 +468,8 @@ module Impl implements RegexTreeViewSig {
    */
   class RegExpCharEscape = RegExpEscape;
 
+  private import codeql.util.Numbers as Numbers
+
   /**
    * An escaped regular expression term, that is, a regular expression
    * term starting with a backslash, which is not a backreference.
@@ -528,42 +530,8 @@ module Impl implements RegexTreeViewSig {
      * E.g. for `\u0061` this returns "a".
      */
     private string getUnicode() {
-      exists(int codepoint | codepoint = sum(this.getHexValueFromUnicode(_)) |
-        result = codepoint.toUnicode()
-      )
+      result = Numbers::parseHexInt(this.getText().suffix(2)).toUnicode()
     }
-
-    /**
-     * Gets int value for the `index`th char in the hex number of the unicode escape.
-     * E.g. for `\u0061` and `index = 2` this returns 96 (the number `6` interpreted as hex).
-     */
-    private int getHexValueFromUnicode(int index) {
-      this.isUnicode() and
-      exists(string hex, string char | hex = this.getText().suffix(2) |
-        char = hex.charAt(index) and
-        result = 16.pow(hex.length() - index - 1) * toHex(char)
-      )
-    }
-  }
-
-  /**
-   * Gets the hex number for the `hex` char.
-   */
-  private int toHex(string hex) {
-    hex = [0 .. 9].toString() and
-    result = hex.toInt()
-    or
-    result = 10 and hex = ["a", "A"]
-    or
-    result = 11 and hex = ["b", "B"]
-    or
-    result = 12 and hex = ["c", "C"]
-    or
-    result = 13 and hex = ["d", "D"]
-    or
-    result = 14 and hex = ["e", "E"]
-    or
-    result = 15 and hex = ["f", "F"]
   }
 
   /**

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll

@@ -1145,19 +1145,13 @@ module Impl<FullStateConfigSig Config> {
       import Param
 
       /* Begin: Stage logic. */
-      // use an alias as a workaround for bad functionality-induced joins
-      pragma[nomagic]
-      private predicate revFlowApAlias(NodeEx node, ApApprox apa) {
-        PrevStage::revFlowAp(node, apa)
-      }
-
       pragma[nomagic]
       private predicate flowIntoCallApa(
         DataFlowCall call, ArgNodeEx arg, ParamNodeEx p, boolean allowsFieldFlow, ApApprox apa
       ) {
         flowIntoCall(call, arg, p, allowsFieldFlow) and
         PrevStage::revFlowAp(p, pragma[only_bind_into](apa)) and
-        revFlowApAlias(arg, pragma[only_bind_into](apa))
+        PrevStage::revFlowAp(arg, pragma[only_bind_into](apa))
       }
 
       pragma[nomagic]
@@ -1167,7 +1161,7 @@ module Impl<FullStateConfigSig Config> {
       ) {
         flowOutOfCall(call, ret, kind, out, allowsFieldFlow) and
         PrevStage::revFlowAp(out, pragma[only_bind_into](apa)) and
-        revFlowApAlias(ret, pragma[only_bind_into](apa))
+        PrevStage::revFlowAp(ret, pragma[only_bind_into](apa))
       }
 
       pragma[nomagic]
@@ -1695,16 +1689,6 @@ module Impl<FullStateConfigSig Config> {
       pragma[nomagic]
       predicate revFlowAp(NodeEx node, Ap ap) { revFlow(node, _, _, _, ap) }
 
-      // use an alias as a workaround for bad functionality-induced joins
-      pragma[nomagic]
-      additional predicate revFlowAlias(NodeEx node) { revFlow(node, _, _, _, _) }
-
-      // use an alias as a workaround for bad functionality-induced joins
-      pragma[nomagic]
-      additional predicate revFlowAlias(NodeEx node, FlowState state, Ap ap) {
-        revFlow(node, state, ap)
-      }
-
       private predicate fwdConsCand(TypedContent tc, Ap ap) { storeStepFwd(_, ap, tc, _, _) }
 
       private predicate revConsCand(TypedContent tc, Ap ap) { storeStepCand(_, ap, tc, _, _) }
@@ -1978,7 +1962,7 @@ module Impl<FullStateConfigSig Config> {
   ) {
     flowOutOfCallNodeCand1(call, node1, kind, node2, allowsFieldFlow) and
     Stage2::revFlow(node2) and
-    Stage2::revFlowAlias(node1)
+    Stage2::revFlow(node1)
   }
 
   pragma[nomagic]
@@ -1987,7 +1971,7 @@ module Impl<FullStateConfigSig Config> {
   ) {
     flowIntoCallNodeCand1(call, node1, node2, allowsFieldFlow) and
     Stage2::revFlow(node2) and
-    Stage2::revFlowAlias(node1)
+    Stage2::revFlow(node1)
   }
 
   private module LocalFlowBigStep {
@@ -2069,11 +2053,11 @@ module Impl<FullStateConfigSig Config> {
       additionalLocalFlowStepNodeCand1(node1, node2) and
       state1 = state2 and
       Stage2::revFlow(node1, pragma[only_bind_into](state1), false) and
-      Stage2::revFlowAlias(node2, pragma[only_bind_into](state2), false)
+      Stage2::revFlow(node2, pragma[only_bind_into](state2), false)
       or
       additionalLocalStateStep(node1, state1, node2, state2) and
       Stage2::revFlow(node1, state1, false) and
-      Stage2::revFlowAlias(node2, state2, false)
+      Stage2::revFlow(node2, state2, false)
     }
 
     /**
@@ -2266,7 +2250,7 @@ module Impl<FullStateConfigSig Config> {
     ) {
       localFlowBigStep(node1, state1, node2, state2, preservesValue, ap.getType(), _) and
       PrevStage::revFlow(node1, pragma[only_bind_into](state1), _) and
-      PrevStage::revFlowAlias(node2, pragma[only_bind_into](state2), _) and
+      PrevStage::revFlow(node2, pragma[only_bind_into](state2), _) and
       exists(lcc)
     }
 
@@ -2277,7 +2261,7 @@ module Impl<FullStateConfigSig Config> {
       exists(FlowState state |
         flowOutOfCallNodeCand2(call, node1, kind, node2, allowsFieldFlow) and
         PrevStage::revFlow(node2, pragma[only_bind_into](state), _) and
-        PrevStage::revFlowAlias(node1, pragma[only_bind_into](state), _)
+        PrevStage::revFlow(node1, pragma[only_bind_into](state), _)
       )
     }
 
@@ -2288,7 +2272,7 @@ module Impl<FullStateConfigSig Config> {
       exists(FlowState state |
         flowIntoCallNodeCand2(call, node1, node2, allowsFieldFlow) and
         PrevStage::revFlow(node2, pragma[only_bind_into](state), _) and
-        PrevStage::revFlowAlias(node1, pragma[only_bind_into](state), _)
+        PrevStage::revFlow(node1, pragma[only_bind_into](state), _)
       )
     }
 
@@ -2590,7 +2574,7 @@ module Impl<FullStateConfigSig Config> {
     ) {
       localFlowBigStep(node1, state1, node2, state2, preservesValue, ap.getType(), lcc) and
       PrevStage::revFlow(node1, pragma[only_bind_into](state1), _) and
-      PrevStage::revFlowAlias(node2, pragma[only_bind_into](state2), _)
+      PrevStage::revFlow(node2, pragma[only_bind_into](state2), _)
     }
 
     pragma[nomagic]
@@ -2600,7 +2584,7 @@ module Impl<FullStateConfigSig Config> {
       exists(FlowState state |
         flowOutOfCallNodeCand2(call, node1, kind, node2, allowsFieldFlow) and
         PrevStage::revFlow(node2, pragma[only_bind_into](state), _) and
-        PrevStage::revFlowAlias(node1, pragma[only_bind_into](state), _)
+        PrevStage::revFlow(node1, pragma[only_bind_into](state), _)
       )
     }
 
@@ -2611,7 +2595,7 @@ module Impl<FullStateConfigSig Config> {
       exists(FlowState state |
         flowIntoCallNodeCand2(call, node1, node2, allowsFieldFlow) and
         PrevStage::revFlow(node2, pragma[only_bind_into](state), _) and
-        PrevStage::revFlowAlias(node1, pragma[only_bind_into](state), _)
+        PrevStage::revFlow(node1, pragma[only_bind_into](state), _)
       )
     }
 

python/ql/lib/semmle/python/dataflow/new/internal/FlowSummaryImpl.qll

@@ -1050,6 +1050,16 @@ module Private {
       not exists(interpretComponent(c))
     }
 
+    /**
+     * Holds if token `part` of specification `spec` has an invalid index.
+     * E.g., `Argument[-1]`.
+     */
+    predicate invalidIndexComponent(AccessPath spec, AccessPathToken part) {
+      part = spec.getToken(_) and
+      part.getName() = ["Parameter", "Argument"] and
+      AccessPath::parseInt(part.getArgumentList()) < 0
+    }
+
     private predicate inputNeedsReference(AccessPathToken c) {
       c.getName() = "Argument" or
       inputNeedsReferenceSpecific(c)

python/ql/lib/semmle/python/regex.qll

@@ -140,7 +140,11 @@ string mode_from_node(DataFlow::Node node) { node = re_flag_tracker(result) }
 
 /** A StrConst used as a regular expression */
 abstract class RegexString extends Expr {
-  RegexString() { (this instanceof Bytes or this instanceof Unicode) }
+  RegexString() {
+    (this instanceof Bytes or this instanceof Unicode) and
+    // is part of the user code
+    exists(this.getLocation().getFile().getRelativePath())
+  }
 
   /**
    * Helper predicate for `char_set_start(int start, int end)`.

python/ql/src/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.6.5
+
+### New Queries
+
+* Added a new query, `py/shell-command-constructed-from-input`, to detect libraries that unsafely construct shell commands from their inputs.
+
 ## 0.6.4
 
 No user-facing changes.

python/ql/src/change-notes/released/0.6.5.md

@@ -1,4 +1,5 @@
----
-category: newQuery
----
+## 0.6.5
+
+### New Queries
+
 * Added a new query, `py/shell-command-constructed-from-input`, to detect libraries that unsafely construct shell commands from their inputs.

python/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.6.4
+lastReleaseVersion: 0.6.5

python/ql/src/qlpack.yml

@@ -1,12 +1,11 @@
 name: codeql/python-queries
-version: 0.6.5-dev
+version: 0.6.6-dev
 groups: 
   - python
   - queries
 dependencies:
   codeql/python-all: ${workspace}
   codeql/suite-helpers: ${workspace}
-  codeql/util: ${workspace}
 suites: codeql-suites
 extractor: python
 defaultSuiteFile: codeql-suites/python-code-scanning.qls

python/ql/test/experimental/dataflow/variable-capture/dataflow-consistency.expected

@@ -1,5 +1,10 @@
 uniqueEnclosingCallable
 uniqueCallEnclosingCallable
+| collections.py:39:17:39:38 | Lambda() | Call should have one enclosing callable but has 0. |
+| collections.py:39:17:39:38 | Lambda() | Call should have one enclosing callable but has 0. |
+| collections.py:45:19:45:24 | mod() | Call should have one enclosing callable but has 0. |
+| collections.py:45:19:45:24 | mod() | Call should have one enclosing callable but has 0. |
+| collections.py:52:13:52:24 | mod_local() | Call should have one enclosing callable but has 0. |
 uniqueType
 uniqueNodeLocation
 missingLocation

python/ql/test/query-tests/Security/CWE-730-PolynomialReDoS/PolynomialReDoS.expected

@@ -7,6 +7,7 @@ edges
 | test.py:7:12:7:23 | ControlFlowNode for Attribute | test.py:9:32:9:35 | ControlFlowNode for text |
 | test.py:7:12:7:23 | ControlFlowNode for Attribute | test.py:12:17:12:20 | ControlFlowNode for text |
 | test.py:7:12:7:23 | ControlFlowNode for Attribute | test.py:18:28:18:31 | ControlFlowNode for text |
+| test.py:7:12:7:23 | ControlFlowNode for Attribute | test.py:21:18:21:21 | ControlFlowNode for text |
 | test.py:14:33:14:39 | ControlFlowNode for my_text | test.py:16:24:16:30 | ControlFlowNode for my_text |
 | test.py:18:28:18:31 | ControlFlowNode for text | test.py:14:33:14:39 | ControlFlowNode for my_text |
 nodes
@@ -21,6 +22,7 @@ nodes
 | test.py:14:33:14:39 | ControlFlowNode for my_text | semmle.label | ControlFlowNode for my_text |
 | test.py:16:24:16:30 | ControlFlowNode for my_text | semmle.label | ControlFlowNode for my_text |
 | test.py:18:28:18:31 | ControlFlowNode for text | semmle.label | ControlFlowNode for text |
+| test.py:21:18:21:21 | ControlFlowNode for text | semmle.label | ControlFlowNode for text |
 subpaths
 #select
 | test.py:8:30:8:33 | ControlFlowNode for text | test.py:2:26:2:32 | ControlFlowNode for ImportMember | test.py:8:30:8:33 | ControlFlowNode for text | This $@ that depends on a $@ may run slow on strings with many repetitions of ' '. | test.py:8:21:8:23 | \\s+ | regular expression | test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |

python/ql/test/query-tests/Security/CWE-730-PolynomialReDoS/test.py

@@ -17,3 +17,6 @@ def code_execution():
 
     indirect(r"^\s+|\s+$", text)
 
+    reg2 = re.compile(r"(AA|BB)(AA|BB)(AA|BB)(AA|BB)(AA|BB)(AA|BB)(AA|BB)(AA|BB)(AA|BB)(AA|BB)(AA|BB)(AA|BB)(AA|BB)(AA|BB)(AA|BB)(AA|BB)(AA|BB)(AA|BB)(AA|BB)(AA|BB)(AA|BB)(AA|BB)(AA|BB)(AA|BB)C.*")
+    reg2.sub("", text) # NOT OK
+