Merge pull request #9712 from erik-krogh/badRange

JS/RB/PY/Java: add suspicious range query

config/identical-files.json

@@ -507,6 +507,12 @@
     "python/ql/lib/semmle/python/security/BadTagFilterQuery.qll",
     "ruby/ql/lib/codeql/ruby/security/BadTagFilterQuery.qll"
   ],
+  "OverlyLargeRange Python/JS/Ruby/Java": [
+    "javascript/ql/lib/semmle/javascript/security/OverlyLargeRangeQuery.qll",
+    "python/ql/lib/semmle/python/security/OverlyLargeRangeQuery.qll",
+    "ruby/ql/lib/codeql/ruby/security/OverlyLargeRangeQuery.qll",
+    "java/ql/lib/semmle/code/java/security/OverlyLargeRangeQuery.qll"
+  ],
   "CFG": [
     "csharp/ql/lib/semmle/code/csharp/controlflow/internal/ControlFlowGraphImplShared.qll",
     "ruby/ql/lib/codeql/ruby/controlflow/internal/ControlFlowGraphImplShared.qll",

java/ql/lib/semmle/code/java/security/OverlyLargeRangeQuery.qll

@@ -0,0 +1,281 @@
+/**
+ * Classes and predicates for working with suspicious character ranges.
+ */
+
+// We don't need the NFA utils, just the regexp tree.
+// but the below is a nice shared library that exposes the API we need.
+import performance.ReDoSUtil
+
+/**
+ * Gets a rank for `range` that is unique for ranges in the same file.
+ * Prioritizes ranges that match more characters.
+ */
+int rankRange(RegExpCharacterRange range) {
+  range =
+    rank[result](RegExpCharacterRange r, Location l, int low, int high |
+      r.getLocation() = l and
+      isRange(r, low, high)
+    |
+      r order by (high - low) desc, l.getStartLine(), l.getStartColumn()
+    )
+}
+
+/** Holds if `range` spans from the unicode code points `low` to `high` (both inclusive). */
+predicate isRange(RegExpCharacterRange range, int low, int high) {
+  exists(string lowc, string highc |
+    range.isRange(lowc, highc) and
+    low.toUnicode() = lowc and
+    high.toUnicode() = highc
+  )
+}
+
+/** Holds if `char` is an alpha-numeric character. */
+predicate isAlphanumeric(string char) {
+  // written like this to avoid having a bindingset for the predicate
+  char = [[48 .. 57], [65 .. 90], [97 .. 122]].toUnicode() // 0-9, A-Z, a-z
+}
+
+/**
+ * Holds if the given ranges are from the same character class
+ * and there exists at least one character matched by both ranges.
+ */
+predicate overlap(RegExpCharacterRange a, RegExpCharacterRange b) {
+  exists(RegExpCharacterClass clz |
+    a = clz.getAChild() and
+    b = clz.getAChild() and
+    a != b
+  |
+    exists(int alow, int ahigh, int blow, int bhigh |
+      isRange(a, alow, ahigh) and
+      isRange(b, blow, bhigh) and
+      alow <= bhigh and
+      blow <= ahigh
+    )
+  )
+}
+
+/**
+ * Holds if `range` overlaps with the char class `escape` from the same character class.
+ */
+predicate overlapsWithCharEscape(RegExpCharacterRange range, RegExpCharacterClassEscape escape) {
+  exists(RegExpCharacterClass clz, string low, string high |
+    range = clz.getAChild() and
+    escape = clz.getAChild() and
+    range.isRange(low, high)
+  |
+    escape.getValue() = "w" and
+    getInRange(low, high).regexpMatch("\\w")
+    or
+    escape.getValue() = "d" and
+    getInRange(low, high).regexpMatch("\\d")
+    or
+    escape.getValue() = "s" and
+    getInRange(low, high).regexpMatch("\\s")
+  )
+}
+
+/** Gets the unicode code point for a `char`. */
+bindingset[char]
+int toCodePoint(string char) { result.toUnicode() = char }
+
+/** A character range that appears to be overly wide. */
+class OverlyWideRange extends RegExpCharacterRange {
+  OverlyWideRange() {
+    exists(int low, int high, int numChars |
+      isRange(this, low, high) and
+      numChars = (1 + high - low) and
+      this.getRootTerm().isUsedAsRegExp() and
+      numChars >= 10
+    |
+      // across the Z-a range (which includes backticks)
+      toCodePoint("Z") >= low and
+      toCodePoint("a") <= high
+      or
+      // across the 9-A range (which includes e.g. ; and ?)
+      toCodePoint("9") >= low and
+      toCodePoint("A") <= high
+      or
+      // a non-alphanumeric char as part of the range boundaries
+      exists(int bound | bound = [low, high] | not isAlphanumeric(bound.toUnicode()))
+    ) and
+    // allowlist for known ranges
+    not this = allowedWideRanges()
+  }
+
+  /** Gets a string representation of a character class that matches the same chars as this range. */
+  string printEquivalent() { result = RangePrinter::printEquivalentCharClass(this) }
+}
+
+/** Gets a range that should not be reported as an overly wide range. */
+RegExpCharacterRange allowedWideRanges() {
+  // ~ is the last printable ASCII character, it's used right in various wide ranges.
+  result.isRange(_, "~")
+  or
+  // the same with " " and "!". " " is the first printable character, and "!" is the first non-white-space printable character.
+  result.isRange([" ", "!"], _)
+  or
+  // the `[@-_]` range is intentional
+  result.isRange("@", "_")
+  or
+  // starting from the zero byte is a good indication that it's purposely matching a large range.
+  result.isRange(0.toUnicode(), _)
+}
+
+/** Gets a char between (and including) `low` and `high`. */
+bindingset[low, high]
+private string getInRange(string low, string high) {
+  result = [toCodePoint(low) .. toCodePoint(high)].toUnicode()
+}
+
+/** A module computing an equivalent character class for an overly wide range. */
+module RangePrinter {
+  bindingset[char]
+  bindingset[result]
+  private string next(string char) {
+    exists(int prev, int next |
+      prev.toUnicode() = char and
+      next.toUnicode() = result and
+      next = prev + 1
+    )
+  }
+
+  /** Gets the points where the parts of the pretty printed range should be cut off. */
+  private string cutoffs() { result = ["A", "Z", "a", "z", "0", "9"] }
+
+  /** Gets the char to use in the low end of a range for a given `cut` */
+  private string lowCut(string cut) {
+    cut = ["A", "a", "0"] and
+    result = cut
+    or
+    cut = ["Z", "z", "9"] and
+    result = next(cut)
+  }
+
+  /** Gets the char to use in the high end of a range for a given `cut` */
+  private string highCut(string cut) {
+    cut = ["Z", "z", "9"] and
+    result = cut
+    or
+    cut = ["A", "a", "0"] and
+    next(result) = cut
+  }
+
+  /** Gets the cutoff char used for a given `part` of a range when pretty-printing it. */
+  private string cutoff(OverlyWideRange range, int part) {
+    exists(int low, int high | isRange(range, low, high) |
+      result =
+        rank[part + 1](string cut |
+          cut = cutoffs() and low < toCodePoint(cut) and toCodePoint(cut) < high
+        |
+          cut order by toCodePoint(cut)
+        )
+    )
+  }
+
+  /** Gets the number of parts we should print for a given `range`. */
+  private int parts(OverlyWideRange range) { result = 1 + strictcount(cutoff(range, _)) }
+
+  /** Holds if the given part of a range should span from `low` to `high`. */
+  private predicate part(OverlyWideRange range, int part, string low, string high) {
+    // first part.
+    part = 0 and
+    (
+      range.isRange(low, high) and
+      parts(range) = 1
+      or
+      parts(range) >= 2 and
+      range.isRange(low, _) and
+      high = highCut(cutoff(range, part))
+    )
+    or
+    // middle
+    part >= 1 and
+    part < parts(range) - 1 and
+    low = lowCut(cutoff(range, part - 1)) and
+    high = highCut(cutoff(range, part))
+    or
+    // last.
+    part = parts(range) - 1 and
+    low = lowCut(cutoff(range, part - 1)) and
+    range.isRange(_, high)
+  }
+
+  /** Gets an escaped `char` for use in a character class. */
+  bindingset[char]
+  private string escape(string char) {
+    exists(string reg | reg = "(\\[|\\]|\\\\|-|/)" |
+      if char.regexpMatch(reg) then result = "\\" + char else result = char
+    )
+  }
+
+  /** Gets a part of the equivalent range. */
+  private string printEquivalentCharClass(OverlyWideRange range, int part) {
+    exists(string low, string high | part(range, part, low, high) |
+      if
+        isAlphanumeric(low) and
+        isAlphanumeric(high)
+      then result = low + "-" + high
+      else
+        result =
+          strictconcat(string char | char = getInRange(low, high) | escape(char) order by char)
+    )
+  }
+
+  /** Gets the entire pretty printed equivalent range. */
+  string printEquivalentCharClass(OverlyWideRange range) {
+    result =
+      strictconcat(string r, int part |
+        r = "[" and part = -1 and exists(range)
+        or
+        r = printEquivalentCharClass(range, part)
+        or
+        r = "]" and part = parts(range)
+      |
+        r order by part
+      )
+  }
+}
+
+/** Gets a char range that is overly large because of `reason`. */
+RegExpCharacterRange getABadRange(string reason, int priority) {
+  priority = 0 and
+  reason = "is equivalent to " + result.(OverlyWideRange).printEquivalent()
+  or
+  priority = 1 and
+  exists(RegExpCharacterRange other |
+    reason = "overlaps with " + other + " in the same character class" and
+    rankRange(result) < rankRange(other) and
+    overlap(result, other)
+  )
+  or
+  priority = 2 and
+  exists(RegExpCharacterClassEscape escape |
+    reason = "overlaps with " + escape + " in the same character class" and
+    overlapsWithCharEscape(result, escape)
+  )
+  or
+  reason = "is empty" and
+  priority = 3 and
+  exists(int low, int high |
+    isRange(result, low, high) and
+    low > high
+  )
+}
+
+/** Holds if `range` matches suspiciously many characters. */
+predicate problem(RegExpCharacterRange range, string reason) {
+  reason =
+    strictconcat(string m, int priority |
+      range = getABadRange(m, priority)
+    |
+      m, ", and " order by priority desc
+    ) and
+  // specifying a range using an escape is usually OK.
+  not range.getAChild() instanceof RegExpEscape and
+  // Unicode escapes in strings are interpreted before it turns into a regexp,
+  // so e.g. [\u0001-\uFFFF] will just turn up as a range between two constants.
+  // We therefore exclude these ranges.
+  range.getRootTerm().getParent() instanceof RegExpLiteral and
+  // is used as regexp (mostly for JS where regular expressions are parsed eagerly)
+  range.getRootTerm().isUsedAsRegExp()
+}

java/ql/src/Security/CWE/CWE-020/OverlyLargeRange.qhelp

@@ -0,0 +1,71 @@
+<!DOCTYPE qhelp PUBLIC
+"-//Semmle//qhelp//EN"
+"qhelp.dtd">
+<qhelp>
+
+    <overview>
+        <p>
+            It's easy to write a regular expression range that matches a wider range of characters than you intended.
+            For example, <code>/[a-zA-z]/</code> matches all lowercase and all uppercase letters,
+            as you would expect, but it also matches the characters: <code>[ \ ] ^ _ `</code>.  
+        </p>
+        <p>
+            Another common problem is failing to escape the dash character in a regular 
+            expression. An unescaped dash is interpreted 
+            as part of a range. For example, in the character class <code>[a-zA-Z0-9%=.,-_]</code> 
+            the last character range matches the 55 characters between 
+            <code>,</code> and <code>_</code> (both included), which overlaps with the 
+            range <code>[0-9]</code> and is clearly not intended by the writer.
+        </p>
+    </overview>
+
+    <recommendation>
+        <p>
+            Avoid any confusion about which characters are included in the range by 
+            writing unambiguous regular expressions.
+            Always check that character ranges match only the expected characters.
+        </p>
+    </recommendation>
+
+    <example>
+
+        <p>
+            The following example code is intended to check whether a string is a valid 6 digit hex color. 
+        </p>
+
+<sample language="java">
+import java.util.regex.Pattern
+public class Tester {
+    public static boolean is_valid_hex_color(String color) {
+        return Pattern.matches("#[0-9a-fA-f]{6}", color);
+    }
+}
+</sample>
+
+        <p>
+            However, the <code>A-f</code> range is overly large and matches every uppercase character. 
+            It would parse a "color" like <code>#XXYYZZ</code> as valid.
+        </p>
+
+        <p>
+            The fix is to use an uppercase <code>A-F</code> range instead.
+        </p>
+
+<sample language="javascript">
+import java.util.regex.Pattern
+public class Tester {
+    public static boolean is_valid_hex_color(String color) {
+        return Pattern.matches("#[0-9a-fA-F]{6}", color);
+    }
+}
+</sample>
+
+    </example>
+
+    <references>
+        <li>GitHub Advisory Database: <a href="https://github.com/advisories/GHSA-g4rg-993r-mgx7">CVE-2021-42740: Improper Neutralization of Special Elements used in a Command in Shell-quote</a></li>
+        <li>wh0.github.io: <a href="https://wh0.github.io/2021/10/28/shell-quote-rce-exploiting.html">Exploiting CVE-2021-42740</a></li>
+        <li>Yosuke Ota: <a href="https://ota-meshi.github.io/eslint-plugin-regexp/rules/no-obscure-range.html">no-obscure-range</a></li>
+        <li>Paul Boyd: <a href="https://pboyd.io/posts/comma-dash-dot/">The regex [,-.]</a></li>
+    </references>
+</qhelp>

java/ql/src/Security/CWE/CWE-020/OverlyLargeRange.ql

@@ -0,0 +1,26 @@
+/**
+ * @name Overly permissive regular expression range
+ * @description Overly permissive regular expression ranges match a wider range of characters than intended.
+ *              This may allow an attacker to bypass a filter or sanitizer.
+ * @kind problem
+ * @problem.severity warning
+ * @security-severity 5.0
+ * @precision high
+ * @id java/overly-large-range
+ * @tags correctness
+ *       security
+ *       external/cwe/cwe-020
+ */
+
+import semmle.code.java.security.OverlyLargeRangeQuery
+
+RegExpCharacterClass potentialMisparsedCharClass() {
+  // nested char classes are currently misparsed
+  result.getAChild().(RegExpNormalChar).getValue() = "["
+}
+
+from RegExpCharacterRange range, string reason
+where
+  problem(range, reason) and
+  not range.getParent() = potentialMisparsedCharClass()
+select range, "Suspicious character range that " + reason + "."

java/ql/src/change-notes/2022-06-24-suspicious-range.md

@@ -0,0 +1,5 @@
+---
+category: newQuery
+---
+* Added a new query, `java/suspicious-regexp-range`, to detect character ranges in regular expressions that seem to match 
+  too many characters.

java/ql/test/query-tests/security/CWE-020/OverlyLargeRangeQuery.expected

@@ -0,0 +1,10 @@
+| SuspiciousRegexpRange.java:5:47:5:49 | 0-9 | Suspicious character range that overlaps with 3-5 in the same character class. |
+| SuspiciousRegexpRange.java:7:49:7:51 | A-z | Suspicious character range that overlaps with A-Z in the same character class, and is equivalent to [A-Z\\[\\\\\\]^_`a-z]. |
+| SuspiciousRegexpRange.java:9:46:9:48 | z-a | Suspicious character range that is empty. |
+| SuspiciousRegexpRange.java:19:56:19:58 | A-f | Suspicious character range that overlaps with a-f in the same character class, and is equivalent to [A-Z\\[\\\\\\]^_`a-f]. |
+| SuspiciousRegexpRange.java:21:48:21:50 | $-` | Suspicious character range that is equivalent to [$%&'()*+,\\-.\\/0-9:;<=>?@A-Z\\[\\\\\\]^_`]. |
+| SuspiciousRegexpRange.java:23:64:23:66 | +-< | Suspicious character range that is equivalent to [+,\\-.\\/0-9:;<]. |
+| SuspiciousRegexpRange.java:25:63:25:65 | .-_ | Suspicious character range that overlaps with 1-9 in the same character class, and is equivalent to [.\\/0-9:;<=>?@A-Z\\[\\\\\\]^_]. |
+| SuspiciousRegexpRange.java:27:52:27:54 | 7-F | Suspicious character range that is equivalent to [7-9:;<=>?@A-F]. |
+| SuspiciousRegexpRange.java:29:56:29:58 | 0-9 | Suspicious character range that overlaps with \\d in the same character class. |
+| SuspiciousRegexpRange.java:31:60:31:62 | .-? | Suspicious character range that overlaps with \\w in the same character class, and is equivalent to [.\\/0-9:;<=>?]. |

java/ql/test/query-tests/security/CWE-020/OverlyLargeRangeQuery.qlref

@@ -0,0 +1 @@
+Security/CWE/CWE-020/OverlyLargeRange.ql

java/ql/test/query-tests/security/CWE-020/SuspiciousRegexpRange.java

@@ -0,0 +1,37 @@
+import java.util.regex.Pattern;
+
+class SuspiciousRegexpRange {
+    void test() {
+        Pattern overlap1 = Pattern.compile("^[0-93-5]*$"); // NOT OK
+
+        Pattern overlap2 = Pattern.compile("[A-ZA-z]*"); // NOT OK
+        
+        Pattern isEmpty = Pattern.compile("^[z-a]*$"); // NOT OK
+        
+        Pattern isAscii = Pattern.compile("^[\\x00-\\x7F]*$"); // OK
+        
+        Pattern printable = Pattern.compile("[!-~]*"); // OK - used to select most printable ASCII characters
+        
+        Pattern codePoints = Pattern.compile("[^\\x21-\\x7E]|[[\\](){}<>/%]*"); // OK
+        
+        Pattern NON_ALPHANUMERIC_REGEXP = Pattern.compile("([^\\#-~| |!])*"); // OK
+        
+        Pattern smallOverlap = Pattern.compile("[0-9a-fA-f]*"); // NOT OK
+        
+        Pattern weirdRange = Pattern.compile("[$-`]*"); // NOT OK
+        
+        Pattern keywordOperator = Pattern.compile("[!\\~\\*\\/%+-<>\\^|=&]*"); // NOT OK
+        
+        Pattern notYoutube = Pattern.compile("youtu.be/[a-z1-9.-_]+"); // NOT OK
+        
+        Pattern numberToLetter = Pattern.compile("[7-F]*"); // NOT OK
+        
+        Pattern overlapsWithClass1 = Pattern.compile("[0-9\\d]*"); // NOT OK
+        
+        Pattern overlapsWithClass2 = Pattern.compile("[\\w,.-?:*+]*"); // NOT OK
+
+        Pattern nested = Pattern.compile("[[A-Za-z_][A-Za-z0-9._-]]*"); // OK, the dash it at the end
+
+        Pattern octal = Pattern.compile("[\000-\037\040-\045]*"); // OK
+    }
+}

javascript/ql/lib/semmle/javascript/security/OverlyLargeRangeQuery.qll

@@ -0,0 +1,281 @@
+/**
+ * Classes and predicates for working with suspicious character ranges.
+ */
+
+// We don't need the NFA utils, just the regexp tree.
+// but the below is a nice shared library that exposes the API we need.
+import performance.ReDoSUtil
+
+/**
+ * Gets a rank for `range` that is unique for ranges in the same file.
+ * Prioritizes ranges that match more characters.
+ */
+int rankRange(RegExpCharacterRange range) {
+  range =
+    rank[result](RegExpCharacterRange r, Location l, int low, int high |
+      r.getLocation() = l and
+      isRange(r, low, high)
+    |
+      r order by (high - low) desc, l.getStartLine(), l.getStartColumn()
+    )
+}
+
+/** Holds if `range` spans from the unicode code points `low` to `high` (both inclusive). */
+predicate isRange(RegExpCharacterRange range, int low, int high) {
+  exists(string lowc, string highc |
+    range.isRange(lowc, highc) and
+    low.toUnicode() = lowc and
+    high.toUnicode() = highc
+  )
+}
+
+/** Holds if `char` is an alpha-numeric character. */
+predicate isAlphanumeric(string char) {
+  // written like this to avoid having a bindingset for the predicate
+  char = [[48 .. 57], [65 .. 90], [97 .. 122]].toUnicode() // 0-9, A-Z, a-z
+}
+
+/**
+ * Holds if the given ranges are from the same character class
+ * and there exists at least one character matched by both ranges.
+ */
+predicate overlap(RegExpCharacterRange a, RegExpCharacterRange b) {
+  exists(RegExpCharacterClass clz |
+    a = clz.getAChild() and
+    b = clz.getAChild() and
+    a != b
+  |
+    exists(int alow, int ahigh, int blow, int bhigh |
+      isRange(a, alow, ahigh) and
+      isRange(b, blow, bhigh) and
+      alow <= bhigh and
+      blow <= ahigh
+    )
+  )
+}
+
+/**
+ * Holds if `range` overlaps with the char class `escape` from the same character class.
+ */
+predicate overlapsWithCharEscape(RegExpCharacterRange range, RegExpCharacterClassEscape escape) {
+  exists(RegExpCharacterClass clz, string low, string high |
+    range = clz.getAChild() and
+    escape = clz.getAChild() and
+    range.isRange(low, high)
+  |
+    escape.getValue() = "w" and
+    getInRange(low, high).regexpMatch("\\w")
+    or
+    escape.getValue() = "d" and
+    getInRange(low, high).regexpMatch("\\d")
+    or
+    escape.getValue() = "s" and
+    getInRange(low, high).regexpMatch("\\s")
+  )
+}
+
+/** Gets the unicode code point for a `char`. */
+bindingset[char]
+int toCodePoint(string char) { result.toUnicode() = char }
+
+/** A character range that appears to be overly wide. */
+class OverlyWideRange extends RegExpCharacterRange {
+  OverlyWideRange() {
+    exists(int low, int high, int numChars |
+      isRange(this, low, high) and
+      numChars = (1 + high - low) and
+      this.getRootTerm().isUsedAsRegExp() and
+      numChars >= 10
+    |
+      // across the Z-a range (which includes backticks)
+      toCodePoint("Z") >= low and
+      toCodePoint("a") <= high
+      or
+      // across the 9-A range (which includes e.g. ; and ?)
+      toCodePoint("9") >= low and
+      toCodePoint("A") <= high
+      or
+      // a non-alphanumeric char as part of the range boundaries
+      exists(int bound | bound = [low, high] | not isAlphanumeric(bound.toUnicode()))
+    ) and
+    // allowlist for known ranges
+    not this = allowedWideRanges()
+  }
+
+  /** Gets a string representation of a character class that matches the same chars as this range. */
+  string printEquivalent() { result = RangePrinter::printEquivalentCharClass(this) }
+}
+
+/** Gets a range that should not be reported as an overly wide range. */
+RegExpCharacterRange allowedWideRanges() {
+  // ~ is the last printable ASCII character, it's used right in various wide ranges.
+  result.isRange(_, "~")
+  or
+  // the same with " " and "!". " " is the first printable character, and "!" is the first non-white-space printable character.
+  result.isRange([" ", "!"], _)
+  or
+  // the `[@-_]` range is intentional
+  result.isRange("@", "_")
+  or
+  // starting from the zero byte is a good indication that it's purposely matching a large range.
+  result.isRange(0.toUnicode(), _)
+}
+
+/** Gets a char between (and including) `low` and `high`. */
+bindingset[low, high]
+private string getInRange(string low, string high) {
+  result = [toCodePoint(low) .. toCodePoint(high)].toUnicode()
+}
+
+/** A module computing an equivalent character class for an overly wide range. */
+module RangePrinter {
+  bindingset[char]
+  bindingset[result]
+  private string next(string char) {
+    exists(int prev, int next |
+      prev.toUnicode() = char and
+      next.toUnicode() = result and
+      next = prev + 1
+    )
+  }
+
+  /** Gets the points where the parts of the pretty printed range should be cut off. */
+  private string cutoffs() { result = ["A", "Z", "a", "z", "0", "9"] }
+
+  /** Gets the char to use in the low end of a range for a given `cut` */
+  private string lowCut(string cut) {
+    cut = ["A", "a", "0"] and
+    result = cut
+    or
+    cut = ["Z", "z", "9"] and
+    result = next(cut)
+  }
+
+  /** Gets the char to use in the high end of a range for a given `cut` */
+  private string highCut(string cut) {
+    cut = ["Z", "z", "9"] and
+    result = cut
+    or
+    cut = ["A", "a", "0"] and
+    next(result) = cut
+  }
+
+  /** Gets the cutoff char used for a given `part` of a range when pretty-printing it. */
+  private string cutoff(OverlyWideRange range, int part) {
+    exists(int low, int high | isRange(range, low, high) |
+      result =
+        rank[part + 1](string cut |
+          cut = cutoffs() and low < toCodePoint(cut) and toCodePoint(cut) < high
+        |
+          cut order by toCodePoint(cut)
+        )
+    )
+  }
+
+  /** Gets the number of parts we should print for a given `range`. */
+  private int parts(OverlyWideRange range) { result = 1 + strictcount(cutoff(range, _)) }
+
+  /** Holds if the given part of a range should span from `low` to `high`. */
+  private predicate part(OverlyWideRange range, int part, string low, string high) {
+    // first part.
+    part = 0 and
+    (
+      range.isRange(low, high) and
+      parts(range) = 1
+      or
+      parts(range) >= 2 and
+      range.isRange(low, _) and
+      high = highCut(cutoff(range, part))
+    )
+    or
+    // middle
+    part >= 1 and
+    part < parts(range) - 1 and
+    low = lowCut(cutoff(range, part - 1)) and
+    high = highCut(cutoff(range, part))
+    or
+    // last.
+    part = parts(range) - 1 and
+    low = lowCut(cutoff(range, part - 1)) and
+    range.isRange(_, high)
+  }
+
+  /** Gets an escaped `char` for use in a character class. */
+  bindingset[char]
+  private string escape(string char) {
+    exists(string reg | reg = "(\\[|\\]|\\\\|-|/)" |
+      if char.regexpMatch(reg) then result = "\\" + char else result = char
+    )
+  }
+
+  /** Gets a part of the equivalent range. */
+  private string printEquivalentCharClass(OverlyWideRange range, int part) {
+    exists(string low, string high | part(range, part, low, high) |
+      if
+        isAlphanumeric(low) and
+        isAlphanumeric(high)
+      then result = low + "-" + high
+      else
+        result =
+          strictconcat(string char | char = getInRange(low, high) | escape(char) order by char)
+    )
+  }
+
+  /** Gets the entire pretty printed equivalent range. */
+  string printEquivalentCharClass(OverlyWideRange range) {
+    result =
+      strictconcat(string r, int part |
+        r = "[" and part = -1 and exists(range)
+        or
+        r = printEquivalentCharClass(range, part)
+        or
+        r = "]" and part = parts(range)
+      |
+        r order by part
+      )
+  }
+}
+
+/** Gets a char range that is overly large because of `reason`. */
+RegExpCharacterRange getABadRange(string reason, int priority) {
+  priority = 0 and
+  reason = "is equivalent to " + result.(OverlyWideRange).printEquivalent()
+  or
+  priority = 1 and
+  exists(RegExpCharacterRange other |
+    reason = "overlaps with " + other + " in the same character class" and
+    rankRange(result) < rankRange(other) and
+    overlap(result, other)
+  )
+  or
+  priority = 2 and
+  exists(RegExpCharacterClassEscape escape |
+    reason = "overlaps with " + escape + " in the same character class" and
+    overlapsWithCharEscape(result, escape)
+  )
+  or
+  reason = "is empty" and
+  priority = 3 and
+  exists(int low, int high |
+    isRange(result, low, high) and
+    low > high
+  )
+}
+
+/** Holds if `range` matches suspiciously many characters. */
+predicate problem(RegExpCharacterRange range, string reason) {
+  reason =
+    strictconcat(string m, int priority |
+      range = getABadRange(m, priority)
+    |
+      m, ", and " order by priority desc
+    ) and
+  // specifying a range using an escape is usually OK.
+  not range.getAChild() instanceof RegExpEscape and
+  // Unicode escapes in strings are interpreted before it turns into a regexp,
+  // so e.g. [\u0001-\uFFFF] will just turn up as a range between two constants.
+  // We therefore exclude these ranges.
+  range.getRootTerm().getParent() instanceof RegExpLiteral and
+  // is used as regexp (mostly for JS where regular expressions are parsed eagerly)
+  range.getRootTerm().isUsedAsRegExp()
+}

javascript/ql/src/Security/CWE-020/OverlyLargeRange.qhelp

@@ -0,0 +1,65 @@
+<!DOCTYPE qhelp PUBLIC
+"-//Semmle//qhelp//EN"
+"qhelp.dtd">
+<qhelp>
+
+    <overview>
+        <p>
+            It's easy to write a regular expression range that matches a wider range of characters than you intended.
+            For example, <code>/[a-zA-z]/</code> matches all lowercase and all uppercase letters,
+            as you would expect, but it also matches the characters: <code>[ \ ] ^ _ `</code>.  
+        </p>
+        <p>
+            Another common problem is failing to escape the dash character in a regular 
+            expression. An unescaped dash is interpreted 
+            as part of a range. For example, in the character class <code>[a-zA-Z0-9%=.,-_]</code> 
+            the last character range matches the 55 characters between 
+            <code>,</code> and <code>_</code> (both included), which overlaps with the 
+            range <code>[0-9]</code> and is clearly not intended by the writer.
+        </p>
+    </overview>
+
+    <recommendation>
+        <p>
+            Avoid any confusion about which characters are included in the range by 
+            writing unambiguous regular expressions.
+            Always check that character ranges match only the expected characters.
+        </p>
+    </recommendation>
+
+    <example>
+
+        <p>
+            The following example code is intended to check whether a string is a valid 6 digit hex color. 
+        </p>
+
+<sample language="javascript">
+function isValidHexColor(color) {
+    return /^#[0-9a-fA-f]{6}$/i.test(color);
+}
+</sample>
+
+        <p>
+            However, the <code>A-f</code> range is overly large and matches every uppercase character. 
+            It would parse a "color" like <code>#XXYYZZ</code> as valid.
+        </p>
+
+        <p>
+            The fix is to use an uppercase <code>A-F</code> range instead.
+        </p>
+
+<sample language="javascript">
+function isValidHexColor(color) {
+    return /^#[0-9A-F]{6}$/i.test(color);
+}
+</sample>
+
+    </example>
+
+    <references>
+        <li>GitHub Advisory Database: <a href="https://github.com/advisories/GHSA-g4rg-993r-mgx7">CVE-2021-42740: Improper Neutralization of Special Elements used in a Command in Shell-quote</a></li>
+        <li>wh0.github.io: <a href="https://wh0.github.io/2021/10/28/shell-quote-rce-exploiting.html">Exploiting CVE-2021-42740</a></li>
+        <li>Yosuke Ota: <a href="https://ota-meshi.github.io/eslint-plugin-regexp/rules/no-obscure-range.html">no-obscure-range</a></li>
+        <li>Paul Boyd: <a href="https://pboyd.io/posts/comma-dash-dot/">The regex [,-.]</a></li>
+    </references>
+</qhelp>

javascript/ql/src/Security/CWE-020/OverlyLargeRange.ql

@@ -0,0 +1,19 @@
+/**
+ * @name Overly permissive regular expression range
+ * @description Overly permissive regular expression ranges match a wider range of characters than intended.
+ *              This may allow an attacker to bypass a filter or sanitizer.
+ * @kind problem
+ * @problem.severity warning
+ * @security-severity 5.0
+ * @precision high
+ * @id js/overly-large-range
+ * @tags correctness
+ *       security
+ *       external/cwe/cwe-020
+ */
+
+import semmle.javascript.security.OverlyLargeRangeQuery
+
+from RegExpCharacterRange range, string reason
+where problem(range, reason)
+select range, "Suspicious character range that " + reason + "."

javascript/ql/src/change-notes/2022-06-24-suspicious-range.md

@@ -0,0 +1,5 @@
+---
+category: newQuery
+---
+* Added a new query, `py/suspicious-regexp-range`, to detect character ranges in regular expressions that seem to match 
+  too many characters.

javascript/ql/test/query-tests/Security/CWE-020/SuspiciousRegexpRange/OverlyLargeRangeQuery.expected

@@ -0,0 +1,10 @@
+| tst.js:1:19:1:21 | 0-9 | Suspicious character range that overlaps with 3-5 in the same character class. |
+| tst.js:3:21:3:23 | A-z | Suspicious character range that overlaps with A-Z in the same character class, and is equivalent to [A-Z\\[\\\\\\]^_`a-z]. |
+| tst.js:5:18:5:20 | z-a | Suspicious character range that is empty. |
+| tst.js:15:28:15:30 | A-f | Suspicious character range that overlaps with a-f in the same character class, and is equivalent to [A-Z\\[\\\\\\]^_`a-f]. |
+| tst.js:17:20:17:22 | $-` | Suspicious character range that is equivalent to [$%&'()*+,\\-.\\/0-9:;<=>?@A-Z\\[\\\\\\]^_`]. |
+| tst.js:19:33:19:35 | +-< | Suspicious character range that is equivalent to [+,\\-.\\/0-9:;<]. |
+| tst.js:21:37:21:39 | .-_ | Suspicious character range that overlaps with 1-9 in the same character class, and is equivalent to [.\\/0-9:;<=>?@A-Z\\[\\\\\\]^_]. |
+| tst.js:23:24:23:26 | 7-F | Suspicious character range that is equivalent to [7-9:;<=>?@A-F]. |
+| tst.js:25:28:25:30 | 0-9 | Suspicious character range that overlaps with \\d in the same character class. |
+| tst.js:27:31:27:33 | .-? | Suspicious character range that overlaps with \\w in the same character class, and is equivalent to [.\\/0-9:;<=>?]. |

javascript/ql/test/query-tests/Security/CWE-020/SuspiciousRegexpRange/OverlyLargeRangeQuery.qlref

@@ -0,0 +1 @@
+Security/CWE-020/OverlyLargeRange.ql

javascript/ql/test/query-tests/Security/CWE-020/SuspiciousRegexpRange/tst.js

@@ -0,0 +1,27 @@
+var overlap1 = /^[0-93-5]$/; // NOT OK
+
+var overlap2 = /[A-ZA-z]/; // NOT OK
+
+var isEmpty = /^[z-a]$/; // NOT OK
+
+var isAscii = /^[\x00-\x7F]*$/; // OK
+
+var printable = /[!-~]/; // OK - used to select most printable ASCII characters
+
+var codePoints = /[^\x21-\x7E]|[[\](){}<>/%]/g; // OK 
+
+const NON_ALPHANUMERIC_REGEXP = /([^\#-~| |!])/g; // OK
+
+var smallOverlap = /[0-9a-fA-f]/; // NOT OK
+
+var weirdRange = /[$-`]/; // NOT OK
+
+var keywordOperator = /[!\~\*\/%+-<>\^|=&]/; // NOT OK
+
+var notYoutube = /youtu\.be\/[a-z1-9.-_]+/; // NOT OK
+
+var numberToLetter = /[7-F]/; // NOT OK
+
+var overlapsWithClass1 = /[0-9\d]/; // NOT OK
+
+var overlapsWithClass2 = /[\w,.-?:*+]/; // NOT OK

python/ql/lib/semmle/python/security/OverlyLargeRangeQuery.qll

@@ -0,0 +1,281 @@
+/**
+ * Classes and predicates for working with suspicious character ranges.
+ */
+
+// We don't need the NFA utils, just the regexp tree.
+// but the below is a nice shared library that exposes the API we need.
+import performance.ReDoSUtil
+
+/**
+ * Gets a rank for `range` that is unique for ranges in the same file.
+ * Prioritizes ranges that match more characters.
+ */
+int rankRange(RegExpCharacterRange range) {
+  range =
+    rank[result](RegExpCharacterRange r, Location l, int low, int high |
+      r.getLocation() = l and
+      isRange(r, low, high)
+    |
+      r order by (high - low) desc, l.getStartLine(), l.getStartColumn()
+    )
+}
+
+/** Holds if `range` spans from the unicode code points `low` to `high` (both inclusive). */
+predicate isRange(RegExpCharacterRange range, int low, int high) {
+  exists(string lowc, string highc |
+    range.isRange(lowc, highc) and
+    low.toUnicode() = lowc and
+    high.toUnicode() = highc
+  )
+}
+
+/** Holds if `char` is an alpha-numeric character. */
+predicate isAlphanumeric(string char) {
+  // written like this to avoid having a bindingset for the predicate
+  char = [[48 .. 57], [65 .. 90], [97 .. 122]].toUnicode() // 0-9, A-Z, a-z
+}
+
+/**
+ * Holds if the given ranges are from the same character class
+ * and there exists at least one character matched by both ranges.
+ */
+predicate overlap(RegExpCharacterRange a, RegExpCharacterRange b) {
+  exists(RegExpCharacterClass clz |
+    a = clz.getAChild() and
+    b = clz.getAChild() and
+    a != b
+  |
+    exists(int alow, int ahigh, int blow, int bhigh |
+      isRange(a, alow, ahigh) and
+      isRange(b, blow, bhigh) and
+      alow <= bhigh and
+      blow <= ahigh
+    )
+  )
+}
+
+/**
+ * Holds if `range` overlaps with the char class `escape` from the same character class.
+ */
+predicate overlapsWithCharEscape(RegExpCharacterRange range, RegExpCharacterClassEscape escape) {
+  exists(RegExpCharacterClass clz, string low, string high |
+    range = clz.getAChild() and
+    escape = clz.getAChild() and
+    range.isRange(low, high)
+  |
+    escape.getValue() = "w" and
+    getInRange(low, high).regexpMatch("\\w")
+    or
+    escape.getValue() = "d" and
+    getInRange(low, high).regexpMatch("\\d")
+    or
+    escape.getValue() = "s" and
+    getInRange(low, high).regexpMatch("\\s")
+  )
+}
+
+/** Gets the unicode code point for a `char`. */
+bindingset[char]
+int toCodePoint(string char) { result.toUnicode() = char }
+
+/** A character range that appears to be overly wide. */
+class OverlyWideRange extends RegExpCharacterRange {
+  OverlyWideRange() {
+    exists(int low, int high, int numChars |
+      isRange(this, low, high) and
+      numChars = (1 + high - low) and
+      this.getRootTerm().isUsedAsRegExp() and
+      numChars >= 10
+    |
+      // across the Z-a range (which includes backticks)
+      toCodePoint("Z") >= low and
+      toCodePoint("a") <= high
+      or
+      // across the 9-A range (which includes e.g. ; and ?)
+      toCodePoint("9") >= low and
+      toCodePoint("A") <= high
+      or
+      // a non-alphanumeric char as part of the range boundaries
+      exists(int bound | bound = [low, high] | not isAlphanumeric(bound.toUnicode()))
+    ) and
+    // allowlist for known ranges
+    not this = allowedWideRanges()
+  }
+
+  /** Gets a string representation of a character class that matches the same chars as this range. */
+  string printEquivalent() { result = RangePrinter::printEquivalentCharClass(this) }
+}
+
+/** Gets a range that should not be reported as an overly wide range. */
+RegExpCharacterRange allowedWideRanges() {
+  // ~ is the last printable ASCII character, it's used right in various wide ranges.
+  result.isRange(_, "~")
+  or
+  // the same with " " and "!". " " is the first printable character, and "!" is the first non-white-space printable character.
+  result.isRange([" ", "!"], _)
+  or
+  // the `[@-_]` range is intentional
+  result.isRange("@", "_")
+  or
+  // starting from the zero byte is a good indication that it's purposely matching a large range.
+  result.isRange(0.toUnicode(), _)
+}
+
+/** Gets a char between (and including) `low` and `high`. */
+bindingset[low, high]
+private string getInRange(string low, string high) {
+  result = [toCodePoint(low) .. toCodePoint(high)].toUnicode()
+}
+
+/** A module computing an equivalent character class for an overly wide range. */
+module RangePrinter {
+  bindingset[char]
+  bindingset[result]
+  private string next(string char) {
+    exists(int prev, int next |
+      prev.toUnicode() = char and
+      next.toUnicode() = result and
+      next = prev + 1
+    )
+  }
+
+  /** Gets the points where the parts of the pretty printed range should be cut off. */
+  private string cutoffs() { result = ["A", "Z", "a", "z", "0", "9"] }
+
+  /** Gets the char to use in the low end of a range for a given `cut` */
+  private string lowCut(string cut) {
+    cut = ["A", "a", "0"] and
+    result = cut
+    or
+    cut = ["Z", "z", "9"] and
+    result = next(cut)
+  }
+
+  /** Gets the char to use in the high end of a range for a given `cut` */
+  private string highCut(string cut) {
+    cut = ["Z", "z", "9"] and
+    result = cut
+    or
+    cut = ["A", "a", "0"] and
+    next(result) = cut
+  }
+
+  /** Gets the cutoff char used for a given `part` of a range when pretty-printing it. */
+  private string cutoff(OverlyWideRange range, int part) {
+    exists(int low, int high | isRange(range, low, high) |
+      result =
+        rank[part + 1](string cut |
+          cut = cutoffs() and low < toCodePoint(cut) and toCodePoint(cut) < high
+        |
+          cut order by toCodePoint(cut)
+        )
+    )
+  }
+
+  /** Gets the number of parts we should print for a given `range`. */
+  private int parts(OverlyWideRange range) { result = 1 + strictcount(cutoff(range, _)) }
+
+  /** Holds if the given part of a range should span from `low` to `high`. */
+  private predicate part(OverlyWideRange range, int part, string low, string high) {
+    // first part.
+    part = 0 and
+    (
+      range.isRange(low, high) and
+      parts(range) = 1
+      or
+      parts(range) >= 2 and
+      range.isRange(low, _) and
+      high = highCut(cutoff(range, part))
+    )
+    or
+    // middle
+    part >= 1 and
+    part < parts(range) - 1 and
+    low = lowCut(cutoff(range, part - 1)) and
+    high = highCut(cutoff(range, part))
+    or
+    // last.
+    part = parts(range) - 1 and
+    low = lowCut(cutoff(range, part - 1)) and
+    range.isRange(_, high)
+  }
+
+  /** Gets an escaped `char` for use in a character class. */
+  bindingset[char]
+  private string escape(string char) {
+    exists(string reg | reg = "(\\[|\\]|\\\\|-|/)" |
+      if char.regexpMatch(reg) then result = "\\" + char else result = char
+    )
+  }
+
+  /** Gets a part of the equivalent range. */
+  private string printEquivalentCharClass(OverlyWideRange range, int part) {
+    exists(string low, string high | part(range, part, low, high) |
+      if
+        isAlphanumeric(low) and
+        isAlphanumeric(high)
+      then result = low + "-" + high
+      else
+        result =
+          strictconcat(string char | char = getInRange(low, high) | escape(char) order by char)
+    )
+  }
+
+  /** Gets the entire pretty printed equivalent range. */
+  string printEquivalentCharClass(OverlyWideRange range) {
+    result =
+      strictconcat(string r, int part |
+        r = "[" and part = -1 and exists(range)
+        or
+        r = printEquivalentCharClass(range, part)
+        or
+        r = "]" and part = parts(range)
+      |
+        r order by part
+      )
+  }
+}
+
+/** Gets a char range that is overly large because of `reason`. */
+RegExpCharacterRange getABadRange(string reason, int priority) {
+  priority = 0 and
+  reason = "is equivalent to " + result.(OverlyWideRange).printEquivalent()
+  or
+  priority = 1 and
+  exists(RegExpCharacterRange other |
+    reason = "overlaps with " + other + " in the same character class" and
+    rankRange(result) < rankRange(other) and
+    overlap(result, other)
+  )
+  or
+  priority = 2 and
+  exists(RegExpCharacterClassEscape escape |
+    reason = "overlaps with " + escape + " in the same character class" and
+    overlapsWithCharEscape(result, escape)
+  )
+  or
+  reason = "is empty" and
+  priority = 3 and
+  exists(int low, int high |
+    isRange(result, low, high) and
+    low > high
+  )
+}
+
+/** Holds if `range` matches suspiciously many characters. */
+predicate problem(RegExpCharacterRange range, string reason) {
+  reason =
+    strictconcat(string m, int priority |
+      range = getABadRange(m, priority)
+    |
+      m, ", and " order by priority desc
+    ) and
+  // specifying a range using an escape is usually OK.
+  not range.getAChild() instanceof RegExpEscape and
+  // Unicode escapes in strings are interpreted before it turns into a regexp,
+  // so e.g. [\u0001-\uFFFF] will just turn up as a range between two constants.
+  // We therefore exclude these ranges.
+  range.getRootTerm().getParent() instanceof RegExpLiteral and
+  // is used as regexp (mostly for JS where regular expressions are parsed eagerly)
+  range.getRootTerm().isUsedAsRegExp()
+}

python/ql/src/Security/CWE-020/OverlyLargeRange.qhelp

@@ -0,0 +1,65 @@
+<!DOCTYPE qhelp PUBLIC
+"-//Semmle//qhelp//EN"
+"qhelp.dtd">
+<qhelp>
+
+    <overview>
+        <p>
+            It's easy to write a regular expression range that matches a wider range of characters than you intended.
+            For example, <code>/[a-zA-z]/</code> matches all lowercase and all uppercase letters,
+            as you would expect, but it also matches the characters: <code>[ \ ] ^ _ `</code>.  
+        </p>
+        <p>
+            Another common problem is failing to escape the dash character in a regular 
+            expression. An unescaped dash is interpreted 
+            as part of a range. For example, in the character class <code>[a-zA-Z0-9%=.,-_]</code> 
+            the last character range matches the 55 characters between 
+            <code>,</code> and <code>_</code> (both included), which overlaps with the 
+            range <code>[0-9]</code> and is clearly not intended by the writer.
+        </p>
+    </overview>
+
+    <recommendation>
+        <p>
+            Avoid any confusion about which characters are included in the range by 
+            writing unambiguous regular expressions.
+            Always check that character ranges match only the expected characters.
+        </p>
+    </recommendation>
+
+    <example>
+
+        <p>
+            The following example code is intended to check whether a string is a valid 6 digit hex color. 
+        </p>
+
+<sample language="python">
+import re
+def is_valid_hex_color(color):
+    return re.match(r'^#[0-9a-fA-f]{6}$', color) is not None
+</sample>
+
+        <p>
+            However, the <code>A-f</code> range is overly large and matches every uppercase character. 
+            It would parse a "color" like <code>#XXYYZZ</code> as valid.
+        </p>
+
+        <p>
+            The fix is to use an uppercase <code>A-F</code> range instead.
+        </p>
+
+<sample language="python">
+import re
+def is_valid_hex_color(color):
+    return re.match(r'^#[0-9a-fA-F]{6}$', color) is not None
+</sample>
+
+    </example>
+
+    <references>
+        <li>GitHub Advisory Database: <a href="https://github.com/advisories/GHSA-g4rg-993r-mgx7">CVE-2021-42740: Improper Neutralization of Special Elements used in a Command in Shell-quote</a></li>
+        <li>wh0.github.io: <a href="https://wh0.github.io/2021/10/28/shell-quote-rce-exploiting.html">Exploiting CVE-2021-42740</a></li>
+        <li>Yosuke Ota: <a href="https://ota-meshi.github.io/eslint-plugin-regexp/rules/no-obscure-range.html">no-obscure-range</a></li>
+        <li>Paul Boyd: <a href="https://pboyd.io/posts/comma-dash-dot/">The regex [,-.]</a></li>
+    </references>
+</qhelp>

python/ql/src/Security/CWE-020/OverlyLargeRange.ql

@@ -0,0 +1,19 @@
+/**
+ * @name Overly permissive regular expression range
+ * @description Overly permissive regular expression ranges match a wider range of characters than intended.
+ *              This may allow an attacker to bypass a filter or sanitizer.
+ * @kind problem
+ * @problem.severity warning
+ * @security-severity 5.0
+ * @precision high
+ * @id py/overly-large-range
+ * @tags correctness
+ *       security
+ *       external/cwe/cwe-020
+ */
+
+import semmle.python.security.OverlyLargeRangeQuery
+
+from RegExpCharacterRange range, string reason
+where problem(range, reason)
+select range, "Suspicious character range that " + reason + "."

python/ql/src/change-notes/2022-06-24-suspicious-range.md

@@ -0,0 +1,5 @@
+---
+category: newQuery
+---
+* Added a new query, `py/suspicious-regexp-range`, to detect character ranges in regular expressions that seem to match 
+  too many characters.

python/ql/test/query-tests/Security/CWE-020-SuspiciousRegexpRange/OverlyLargeRangeQuery.expected

@@ -0,0 +1,10 @@
+| test.py:3:29:3:31 | 0-9 | Suspicious character range that overlaps with 3-5 in the same character class. |
+| test.py:5:31:5:33 | A-z | Suspicious character range that overlaps with A-Z in the same character class, and is equivalent to [A-Z\\[\\\\\\]^_`a-z]. |
+| test.py:7:28:7:30 | z-a | Suspicious character range that is empty. |
+| test.py:17:38:17:40 | A-f | Suspicious character range that overlaps with a-f in the same character class, and is equivalent to [A-Z\\[\\\\\\]^_`a-f]. |
+| test.py:19:30:19:32 | $-` | Suspicious character range that is equivalent to [$%&'()*+,\\-.\\/0-9:;<=>?@A-Z\\[\\\\\\]^_`]. |
+| test.py:21:43:21:45 | +-< | Suspicious character range that is equivalent to [+,\\-.\\/0-9:;<]. |
+| test.py:23:47:23:49 | .-_ | Suspicious character range that overlaps with 1-9 in the same character class, and is equivalent to [.\\/0-9:;<=>?@A-Z\\[\\\\\\]^_]. |
+| test.py:25:34:25:36 | 7-F | Suspicious character range that is equivalent to [7-9:;<=>?@A-F]. |
+| test.py:27:38:27:40 | 0-9 | Suspicious character range that overlaps with \\d in the same character class. |
+| test.py:29:41:29:43 | .-? | Suspicious character range that overlaps with \\w in the same character class, and is equivalent to [.\\/0-9:;<=>?]. |

python/ql/test/query-tests/Security/CWE-020-SuspiciousRegexpRange/OverlyLargeRangeQuery.qlref

@@ -0,0 +1 @@
+Security/CWE-020/OverlyLargeRange.ql

python/ql/test/query-tests/Security/CWE-020-SuspiciousRegexpRange/test.py

@@ -0,0 +1,29 @@
+import re
+
+overlap1 = re.compile(r'^[0-93-5]$') # NOT OK
+
+overlap2 = re.compile(r'[A-ZA-z]') # NOT OK
+
+isEmpty = re.compile(r'^[z-a]$') # NOT OK
+
+isAscii = re.compile(r'^[\x00-\x7F]*$') # OK
+
+printable = re.compile(r'[!-~]') # OK - used to select most printable ASCII characters
+
+codePoints = re.compile(r'[^\x21-\x7E]|[[\](){}<>/%]') # OK
+
+NON_ALPHANUMERIC_REGEXP = re.compile(r'([^\#-~| |!])') # OK
+
+smallOverlap = re.compile(r'[0-9a-fA-f]') # NOT OK
+
+weirdRange = re.compile(r'[$-`]') # NOT OK
+
+keywordOperator = re.compile(r'[!\~\*\/%+-<>\^|=&]') # NOT OK
+
+notYoutube = re.compile(r'youtu\.be\/[a-z1-9.-_]+') # NOT OK
+
+numberToLetter = re.compile(r'[7-F]') # NOT OK
+
+overlapsWithClass1 = re.compile(r'[0-9\d]') # NOT OK
+
+overlapsWithClass2 = re.compile(r'[\w,.-?:*+]') # NOT OK

ruby/ql/lib/codeql/ruby/security/OverlyLargeRangeQuery.qll

@@ -0,0 +1,281 @@
+/**
+ * Classes and predicates for working with suspicious character ranges.
+ */
+
+// We don't need the NFA utils, just the regexp tree.
+// but the below is a nice shared library that exposes the API we need.
+import performance.ReDoSUtil
+
+/**
+ * Gets a rank for `range` that is unique for ranges in the same file.
+ * Prioritizes ranges that match more characters.
+ */
+int rankRange(RegExpCharacterRange range) {
+  range =
+    rank[result](RegExpCharacterRange r, Location l, int low, int high |
+      r.getLocation() = l and
+      isRange(r, low, high)
+    |
+      r order by (high - low) desc, l.getStartLine(), l.getStartColumn()
+    )
+}
+
+/** Holds if `range` spans from the unicode code points `low` to `high` (both inclusive). */
+predicate isRange(RegExpCharacterRange range, int low, int high) {
+  exists(string lowc, string highc |
+    range.isRange(lowc, highc) and
+    low.toUnicode() = lowc and
+    high.toUnicode() = highc
+  )
+}
+
+/** Holds if `char` is an alpha-numeric character. */
+predicate isAlphanumeric(string char) {
+  // written like this to avoid having a bindingset for the predicate
+  char = [[48 .. 57], [65 .. 90], [97 .. 122]].toUnicode() // 0-9, A-Z, a-z
+}
+
+/**
+ * Holds if the given ranges are from the same character class
+ * and there exists at least one character matched by both ranges.
+ */
+predicate overlap(RegExpCharacterRange a, RegExpCharacterRange b) {
+  exists(RegExpCharacterClass clz |
+    a = clz.getAChild() and
+    b = clz.getAChild() and
+    a != b
+  |
+    exists(int alow, int ahigh, int blow, int bhigh |
+      isRange(a, alow, ahigh) and
+      isRange(b, blow, bhigh) and
+      alow <= bhigh and
+      blow <= ahigh
+    )
+  )
+}
+
+/**
+ * Holds if `range` overlaps with the char class `escape` from the same character class.
+ */
+predicate overlapsWithCharEscape(RegExpCharacterRange range, RegExpCharacterClassEscape escape) {
+  exists(RegExpCharacterClass clz, string low, string high |
+    range = clz.getAChild() and
+    escape = clz.getAChild() and
+    range.isRange(low, high)
+  |
+    escape.getValue() = "w" and
+    getInRange(low, high).regexpMatch("\\w")
+    or
+    escape.getValue() = "d" and
+    getInRange(low, high).regexpMatch("\\d")
+    or
+    escape.getValue() = "s" and
+    getInRange(low, high).regexpMatch("\\s")
+  )
+}
+
+/** Gets the unicode code point for a `char`. */
+bindingset[char]
+int toCodePoint(string char) { result.toUnicode() = char }
+
+/** A character range that appears to be overly wide. */
+class OverlyWideRange extends RegExpCharacterRange {
+  OverlyWideRange() {
+    exists(int low, int high, int numChars |
+      isRange(this, low, high) and
+      numChars = (1 + high - low) and
+      this.getRootTerm().isUsedAsRegExp() and
+      numChars >= 10
+    |
+      // across the Z-a range (which includes backticks)
+      toCodePoint("Z") >= low and
+      toCodePoint("a") <= high
+      or
+      // across the 9-A range (which includes e.g. ; and ?)
+      toCodePoint("9") >= low and
+      toCodePoint("A") <= high
+      or
+      // a non-alphanumeric char as part of the range boundaries
+      exists(int bound | bound = [low, high] | not isAlphanumeric(bound.toUnicode()))
+    ) and
+    // allowlist for known ranges
+    not this = allowedWideRanges()
+  }
+
+  /** Gets a string representation of a character class that matches the same chars as this range. */
+  string printEquivalent() { result = RangePrinter::printEquivalentCharClass(this) }
+}
+
+/** Gets a range that should not be reported as an overly wide range. */
+RegExpCharacterRange allowedWideRanges() {
+  // ~ is the last printable ASCII character, it's used right in various wide ranges.
+  result.isRange(_, "~")
+  or
+  // the same with " " and "!". " " is the first printable character, and "!" is the first non-white-space printable character.
+  result.isRange([" ", "!"], _)
+  or
+  // the `[@-_]` range is intentional
+  result.isRange("@", "_")
+  or
+  // starting from the zero byte is a good indication that it's purposely matching a large range.
+  result.isRange(0.toUnicode(), _)
+}
+
+/** Gets a char between (and including) `low` and `high`. */
+bindingset[low, high]
+private string getInRange(string low, string high) {
+  result = [toCodePoint(low) .. toCodePoint(high)].toUnicode()
+}
+
+/** A module computing an equivalent character class for an overly wide range. */
+module RangePrinter {
+  bindingset[char]
+  bindingset[result]
+  private string next(string char) {
+    exists(int prev, int next |
+      prev.toUnicode() = char and
+      next.toUnicode() = result and
+      next = prev + 1
+    )
+  }
+
+  /** Gets the points where the parts of the pretty printed range should be cut off. */
+  private string cutoffs() { result = ["A", "Z", "a", "z", "0", "9"] }
+
+  /** Gets the char to use in the low end of a range for a given `cut` */
+  private string lowCut(string cut) {
+    cut = ["A", "a", "0"] and
+    result = cut
+    or
+    cut = ["Z", "z", "9"] and
+    result = next(cut)
+  }
+
+  /** Gets the char to use in the high end of a range for a given `cut` */
+  private string highCut(string cut) {
+    cut = ["Z", "z", "9"] and
+    result = cut
+    or
+    cut = ["A", "a", "0"] and
+    next(result) = cut
+  }
+
+  /** Gets the cutoff char used for a given `part` of a range when pretty-printing it. */
+  private string cutoff(OverlyWideRange range, int part) {
+    exists(int low, int high | isRange(range, low, high) |
+      result =
+        rank[part + 1](string cut |
+          cut = cutoffs() and low < toCodePoint(cut) and toCodePoint(cut) < high
+        |
+          cut order by toCodePoint(cut)
+        )
+    )
+  }
+
+  /** Gets the number of parts we should print for a given `range`. */
+  private int parts(OverlyWideRange range) { result = 1 + strictcount(cutoff(range, _)) }
+
+  /** Holds if the given part of a range should span from `low` to `high`. */
+  private predicate part(OverlyWideRange range, int part, string low, string high) {
+    // first part.
+    part = 0 and
+    (
+      range.isRange(low, high) and
+      parts(range) = 1
+      or
+      parts(range) >= 2 and
+      range.isRange(low, _) and
+      high = highCut(cutoff(range, part))
+    )
+    or
+    // middle
+    part >= 1 and
+    part < parts(range) - 1 and
+    low = lowCut(cutoff(range, part - 1)) and
+    high = highCut(cutoff(range, part))
+    or
+    // last.
+    part = parts(range) - 1 and
+    low = lowCut(cutoff(range, part - 1)) and
+    range.isRange(_, high)
+  }
+
+  /** Gets an escaped `char` for use in a character class. */
+  bindingset[char]
+  private string escape(string char) {
+    exists(string reg | reg = "(\\[|\\]|\\\\|-|/)" |
+      if char.regexpMatch(reg) then result = "\\" + char else result = char
+    )
+  }
+
+  /** Gets a part of the equivalent range. */
+  private string printEquivalentCharClass(OverlyWideRange range, int part) {
+    exists(string low, string high | part(range, part, low, high) |
+      if
+        isAlphanumeric(low) and
+        isAlphanumeric(high)
+      then result = low + "-" + high
+      else
+        result =
+          strictconcat(string char | char = getInRange(low, high) | escape(char) order by char)
+    )
+  }
+
+  /** Gets the entire pretty printed equivalent range. */
+  string printEquivalentCharClass(OverlyWideRange range) {
+    result =
+      strictconcat(string r, int part |
+        r = "[" and part = -1 and exists(range)
+        or
+        r = printEquivalentCharClass(range, part)
+        or
+        r = "]" and part = parts(range)
+      |
+        r order by part
+      )
+  }
+}
+
+/** Gets a char range that is overly large because of `reason`. */
+RegExpCharacterRange getABadRange(string reason, int priority) {
+  priority = 0 and
+  reason = "is equivalent to " + result.(OverlyWideRange).printEquivalent()
+  or
+  priority = 1 and
+  exists(RegExpCharacterRange other |
+    reason = "overlaps with " + other + " in the same character class" and
+    rankRange(result) < rankRange(other) and
+    overlap(result, other)
+  )
+  or
+  priority = 2 and
+  exists(RegExpCharacterClassEscape escape |
+    reason = "overlaps with " + escape + " in the same character class" and
+    overlapsWithCharEscape(result, escape)
+  )
+  or
+  reason = "is empty" and
+  priority = 3 and
+  exists(int low, int high |
+    isRange(result, low, high) and
+    low > high
+  )
+}
+
+/** Holds if `range` matches suspiciously many characters. */
+predicate problem(RegExpCharacterRange range, string reason) {
+  reason =
+    strictconcat(string m, int priority |
+      range = getABadRange(m, priority)
+    |
+      m, ", and " order by priority desc
+    ) and
+  // specifying a range using an escape is usually OK.
+  not range.getAChild() instanceof RegExpEscape and
+  // Unicode escapes in strings are interpreted before it turns into a regexp,
+  // so e.g. [\u0001-\uFFFF] will just turn up as a range between two constants.
+  // We therefore exclude these ranges.
+  range.getRootTerm().getParent() instanceof RegExpLiteral and
+  // is used as regexp (mostly for JS where regular expressions are parsed eagerly)
+  range.getRootTerm().isUsedAsRegExp()
+}

ruby/ql/src/change-notes/2022-06-24-suspicious-range.md

@@ -0,0 +1,5 @@
+---
+category: newQuery
+---
+* Added a new query, `rb/suspicious-regexp-range`, to detect character ranges in regular expressions that seem to match 
+  too many characters.

ruby/ql/src/queries/security/cwe-020/OverlyLargeRange.qhelp

@@ -0,0 +1,65 @@
+<!DOCTYPE qhelp PUBLIC
+"-//Semmle//qhelp//EN"
+"qhelp.dtd">
+<qhelp>
+
+    <overview>
+        <p>
+            It's easy to write a regular expression range that matches a wider range of characters than you intended.
+            For example, <code>/[a-zA-z]/</code> matches all lowercase and all uppercase letters,
+            as you would expect, but it also matches the characters: <code>[ \ ] ^ _ `</code>.  
+        </p>
+        <p>
+            Another common problem is failing to escape the dash character in a regular 
+            expression. An unescaped dash is interpreted 
+            as part of a range. For example, in the character class <code>[a-zA-Z0-9%=.,-_]</code> 
+            the last character range matches the 55 characters between 
+            <code>,</code> and <code>_</code> (both included), which overlaps with the 
+            range <code>[0-9]</code> and is clearly not intended by the writer.
+        </p>
+    </overview>
+
+    <recommendation>
+        <p>
+            Avoid any confusion about which characters are included in the range by 
+            writing unambiguous regular expressions.
+            Always check that character ranges match only the expected characters.
+        </p>
+    </recommendation>
+
+    <example>
+
+        <p>
+            The following example code is intended to check whether a string is a valid 6 digit hex color. 
+        </p>
+
+<sample language="ruby">
+def is_valid_hex_color(color)
+    /^#[0-9a-fA-f]{6}$/.match(color)
+end
+</sample>
+
+        <p>
+            However, the <code>A-f</code> range is overly large and matches every uppercase character. 
+            It would parse a "color" like <code>#XXYYZZ</code> as valid.
+        </p>
+
+        <p>
+            The fix is to use an uppercase <code>A-F</code> range instead.
+        </p>
+
+<sample language="ruby">
+def is_valid_hex_color(color)
+    /^#[0-9a-fA-F]{6}$/.match(color)
+end
+</sample>
+
+    </example>
+
+    <references>
+        <li>GitHub Advisory Database: <a href="https://github.com/advisories/GHSA-g4rg-993r-mgx7">CVE-2021-42740: Improper Neutralization of Special Elements used in a Command in Shell-quote</a></li>
+        <li>wh0.github.io: <a href="https://wh0.github.io/2021/10/28/shell-quote-rce-exploiting.html">Exploiting CVE-2021-42740</a></li>
+        <li>Yosuke Ota: <a href="https://ota-meshi.github.io/eslint-plugin-regexp/rules/no-obscure-range.html">no-obscure-range</a></li>
+        <li>Paul Boyd: <a href="https://pboyd.io/posts/comma-dash-dot/">The regex [,-.]</a></li>
+    </references>
+</qhelp>

ruby/ql/src/queries/security/cwe-020/OverlyLargeRange.ql

@@ -0,0 +1,29 @@
+/**
+ * @name Overly permissive regular expression range
+ * @description Overly permissive regular expression ranges match a wider range of characters than intended.
+ *              This may allow an attacker to bypass a filter or sanitizer.
+ * @kind problem
+ * @problem.severity warning
+ * @security-severity 5.0
+ * @precision high
+ * @id rb/overly-large-range
+ * @tags correctness
+ *       security
+ *       external/cwe/cwe-020
+ */
+
+import codeql.ruby.security.OverlyLargeRangeQuery
+
+RegExpCharacterClass potentialMisparsedCharClass() {
+  // some escapes, e.g. [\000-\037] are currently misparsed.
+  result.getAChild().(RegExpNormalChar).getValue() = "\\"
+  or
+  // nested char classes are currently misparsed
+  result.getAChild().(RegExpNormalChar).getValue() = "["
+}
+
+from RegExpCharacterRange range, string reason
+where
+  problem(range, reason) and
+  not range.getParent() = potentialMisparsedCharClass()
+select range, "Suspicious character range that " + reason + "."

ruby/ql/test/query-tests/security/cwe-020/SuspiciousRegexpRange/OverlyLargeRangeQuery.expected

@@ -0,0 +1,10 @@
+| suspicous_regexp_range.rb:1:15:1:17 | 0-9 | Suspicious character range that overlaps with 3-5 in the same character class. |
+| suspicous_regexp_range.rb:3:17:3:19 | A-z | Suspicious character range that overlaps with A-Z in the same character class, and is equivalent to [A-Z\\[\\\\\\]^_`a-z]. |
+| suspicous_regexp_range.rb:5:14:5:16 | z-a | Suspicious character range that is empty. |
+| suspicous_regexp_range.rb:15:24:15:26 | A-f | Suspicious character range that overlaps with a-f in the same character class, and is equivalent to [A-Z\\[\\\\\\]^_`a-f]. |
+| suspicous_regexp_range.rb:17:16:17:18 | $-` | Suspicious character range that is equivalent to [$%&'()*+,\\-.\\/0-9:;<=>?@A-Z\\[\\\\\\]^_`]. |
+| suspicous_regexp_range.rb:19:29:19:31 | +-< | Suspicious character range that is equivalent to [+,\\-.\\/0-9:;<]. |
+| suspicous_regexp_range.rb:21:33:21:35 | .-_ | Suspicious character range that overlaps with 1-9 in the same character class, and is equivalent to [.\\/0-9:;<=>?@A-Z\\[\\\\\\]^_]. |
+| suspicous_regexp_range.rb:23:20:23:22 | 7-F | Suspicious character range that is equivalent to [7-9:;<=>?@A-F]. |
+| suspicous_regexp_range.rb:25:24:25:26 | 0-9 | Suspicious character range that overlaps with \\d in the same character class. |
+| suspicous_regexp_range.rb:27:27:27:29 | .-? | Suspicious character range that overlaps with \\w in the same character class, and is equivalent to [.\\/0-9:;<=>?]. |

ruby/ql/test/query-tests/security/cwe-020/SuspiciousRegexpRange/OverlyLargeRangeQuery.qlref

@@ -0,0 +1 @@
+queries/security/cwe-020/OverlyLargeRange.ql

ruby/ql/test/query-tests/security/cwe-020/SuspiciousRegexpRange/suspicous_regexp_range.rb

@@ -0,0 +1,31 @@
+overlap1 = /^[0-93-5]$/ # NOT OK
+
+overlap2 = /[A-ZA-z]/ # NOT OK
+
+isEmpty = /^[z-a]$/ # NOT OK
+
+isAscii = /^[\x00-\x7F]*$/ # OK
+
+printable = /[!-~]/ # OK - used to select most printable ASCII characters
+
+codePoints = /[^\x21-\x7E]|[\[\](){}<>\/%]/ # OK
+
+NON_ALPHANUMERIC_REGEXP = /([^\#-~| |!])/ # OK
+
+smallOverlap = /[0-9a-fA-f]/ # NOT OK
+
+weirdRange = /[$-`]/ # NOT OK
+
+keywordOperator = /[!\~\*\/%+-<>\^|=&]/ # NOT OK
+
+notYoutube = /youtu\.be\/[a-z1-9.-_]+/ # NOT OK
+
+numberToLetter = /[7-F]/ # NOT OK
+
+overlapsWithClass1 = /[0-9\d]/ # NOT OK
+
+overlapsWithClass2 = /[\w,.-?:*+]/ # NOT OK
+
+escapes = /[\000-\037\047\134\177-\377]/n # OK - they are escapes
+
+nested = /[a-z&&[^a-c]]/ # OK